diff --git a/.github/workflows/build_radioconda.yml b/.github/workflows/build_radioconda.yml index 91479ef..bd3de5b 100644 --- a/.github/workflows/build_radioconda.yml +++ b/.github/workflows/build_radioconda.yml @@ -69,7 +69,7 @@ jobs: env: PLATFORM: ${{ matrix.PLATFORM }} run: | - cp installer_specs/$DISTNAME-$PLATFORM/$DISTNAME-$PLATFORM.txt dist/ + cp installer_specs/$DISTNAME-$PLATFORM.txt dist/ ls -lh dist - name: Test installer (sh) diff --git a/radioconda.yaml b/radioconda.yaml index 41de7b7..d8aa714 100644 --- a/radioconda.yaml +++ b/radioconda.yaml @@ -7,19 +7,13 @@ platforms: - osx-64 - win-64 dependencies: - - mamba - - ipython - - python - # restrict to python 3.8 on Windows for Windows 7 compatibility - - python 3.8.* # [win] - - radioconda_console_shortcut # [win] - - digital_rf - gnuradio 3.8.* - gnuradio-osmosdr - gnuradio-satellites - gnuradio-soapy - gqrx + - ipython - libiio - libm2k - limesuite @@ -27,6 +21,10 @@ dependencies: - numpy - pandas - pyadi-iio + - python + # restrict to python 3.8 on Windows for Windows 7 compatibility + - python 3.8.* # [win] + - radioconda_console_shortcut # [win] - rtl-sdr - scipy - soapysdr diff --git a/rerender.py b/rerender.py index 00d6d38..6b605a4 100755 --- a/rerender.py +++ b/rerender.py @@ -1,12 +1,17 @@ #!/usr/bin/env python3 import pathlib import shutil +from typing import List, Optional import conda_lock import yaml -def render_lock_spec( +def name_from_pkg_spec(spec: str): + return spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0] + + +def lock_env_spec( lock_spec: conda_lock.src_parser.LockSpecification, conda_exe: str ) -> conda_lock.src_parser.LockSpecification: create_env_dict = conda_lock.conda_lock.solve_specs_for_arch( @@ -16,36 +21,104 @@ def render_lock_spec( platform=lock_spec.platform, ) pkgs = create_env_dict["actions"]["LINK"] - spec_names = set( - spec.split(sep=None, maxsplit=1)[0].split(sep="=", maxsplit=1)[0] - for spec in lock_spec.specs - ) + locked_specs = ["{name}={version}={build_string}".format(**pkg) for pkg in pkgs] - rendered_specs = [] - rendered_dep_specs = [] - for pkg in pkgs: - pkg_spec = "{name}={version}={build_string}".format(**pkg) - if pkg["name"] in spec_names: - rendered_specs.append(pkg_spec) - else: - rendered_dep_specs.append(pkg_spec) - - rendered_lock_spec = conda_lock.src_parser.LockSpecification( - specs=sorted(rendered_specs), - channels=lock_spec.channels, - platform=lock_spec.platform, - ) - rendered_full_lock_spec = conda_lock.src_parser.LockSpecification( - specs=sorted(rendered_specs + rendered_dep_specs), + locked_env_spec = conda_lock.src_parser.LockSpecification( + specs=sorted(locked_specs), channels=lock_spec.channels, platform=lock_spec.platform, ) - return rendered_lock_spec, rendered_full_lock_spec + return locked_env_spec -def render_constructor_specs( +def write_lock_file( + lock_spec: conda_lock.src_parser.LockSpecification, + lock_file_path: pathlib.Path, + name: Optional[str] = None, + version: Optional[str] = None, + channels: Optional[List[str]] = None, +): + lockfile_contents = [ + f"# platform: {lock_spec.platform}", + f"# env_hash: {lock_spec.env_hash()}", + ] + if name: + lockfile_contents.append(f"# name: {name}") + if version: + lockfile_contents.append(f"# version: {version}") + if channels: + lockfile_contents.append(f"# channels: {','.join(channels)}") + lockfile_contents.extend(lock_spec.specs) + with lock_file_path.open("w") as f: + f.write("\n".join(lockfile_contents)) + + +def render_constructor( + lock_spec: conda_lock.src_parser.LockSpecification, + name: str, + version: str, + company: str, + license_file: pathlib.Path, + output_dir: pathlib.Path, +) -> dict: + platform = lock_spec.platform + constructor_name = f"{name}-{platform}" + + construct_dict = dict( + name=name, + version=version, + company=company, + channels=lock_spec.channels, + specs=lock_spec.specs, + initialize_by_default=True, + installer_type="all", + keep_pkgs=True, + license_file="LICENSE", + register_python_default=False, + write_condarc=True, + ) + if platform.startswith("win"): + construct_dict["post_install"] = "post_install.bat" + else: + construct_dict["post_install"] = "post_install.sh" + + constructor_dir = output_dir / constructor_name + if constructor_dir.exists(): + shutil.rmtree(constructor_dir) + constructor_dir.mkdir(parents=True) + + # copy license to the constructor directory + shutil.copy(license_file, constructor_dir / "LICENSE") + + # write the post_install scripts referenced in the construct dict + if platform.startswith("win"): + with (constructor_dir / "post_install.bat").open("w") as f: + f.write("\n".join((r"del /q %PREFIX%\pkgs\*.tar.bz2", "exit 0", ""))) + else: + with (constructor_dir / "post_install.sh").open("w") as f: + f.write( + "\n".join( + ( + "#!/bin/sh", + f'PREFIX="${{PREFIX:-$2/{name}}}"', + r"rm -f $PREFIX/pkgs/*.tar.bz2", + "exit 0", + "", + ) + ) + ) + + construct_yaml_path = constructor_dir / "construct.yaml" + with construct_yaml_path.open("w") as f: + yaml.safe_dump(construct_dict, stream=f) + + return construct_dict + + +def render_platforms( environment_file: pathlib.Path, + installer_pkg_specs: List[str], version: str, company: str, license_file: pathlib.Path, @@ -55,85 +128,83 @@ def render_constructor_specs( with environment_file.open("r") as f: env_yaml_data = yaml.safe_load(f) - installer_name = env_yaml_data["name"] + env_name = env_yaml_data["name"] platforms = env_yaml_data["platforms"] if not license_file.exists(): raise ValueError(f"Cannot find license file: {license_file}") - output_dir.mkdir(parents=True, exist_ok=True) + if output_dir.exists(): + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True) - constructor_specs = {} + rendered_platforms = {} for platform in platforms: - constructor_name = f"{installer_name}-{platform}" + output_name = f"{env_name}-{platform}" - lock_spec = conda_lock.conda_lock.parse_environment_file( + # get the environment specification for the list of packages from the env file + env_spec = conda_lock.conda_lock.parse_environment_file( environment_file=environment_file, platform=platform ) - rendered_lock_spec, rendered_full_lock_spec = render_lock_spec( - lock_spec, conda_exe + + # lock the full environment specification to specific versions and builds + locked_env_spec = lock_env_spec(env_spec, conda_exe) + + # write the full environment specification to a lock file + lock_file_path = output_dir / f"{output_name}.txt" + write_lock_file( + locked_env_spec, + lock_file_path, + name=env_name, + version=version, + channels=locked_env_spec.channels, ) - construct_dict = dict( - name=installer_name, + + # add installer-only (base environment) packages and lock those too + installer_spec = conda_lock.src_parser.LockSpecification( + specs=sorted(locked_env_spec.specs + installer_pkg_specs), + channels=locked_env_spec.channels, + platform=locked_env_spec.platform, + ) + locked_installer_spec = lock_env_spec(installer_spec, conda_exe) + + # get a set of only the packages to put in the constructor specification + # taken from the installer-only list and those explicitly selected originally + constructor_pkg_names = set( + name_from_pkg_spec(spec) for spec in env_spec.specs + installer_pkg_specs + ) + + # filter the installer spec by the constructor package names + constructor_pkg_specs = [ + spec + for spec in locked_installer_spec.specs + if name_from_pkg_spec(spec) in constructor_pkg_names + ] + constructor_spec = conda_lock.src_parser.LockSpecification( + specs=constructor_pkg_specs, + channels=locked_installer_spec.channels, + platform=locked_installer_spec.platform, + ) + + # create the rendered constructor directory + constructor_dict = render_constructor( + lock_spec=constructor_spec, + name=env_name, version=version, company=company, - channels=rendered_lock_spec.channels, - specs=rendered_lock_spec.specs, - initialize_by_default=True, - installer_type="all", - keep_pkgs=True, - license_file="LICENSE", - register_python_default=False, - write_condarc=True, + license_file=license_file, + output_dir=output_dir, ) - if platform.startswith("win"): - construct_dict["post_install"] = "post_install.bat" - else: - construct_dict["post_install"] = "post_install.sh" - constructor_specs[constructor_name] = construct_dict + # aggregate output + rendered_platforms[output_name] = dict( + locked_env_spec=locked_env_spec, + locked_installer_spec=locked_installer_spec, + constructor_dict=constructor_dict, + ) - constructor_dir = output_dir / constructor_name - if constructor_dir.exists(): - shutil.rmtree(constructor_dir) - constructor_dir.mkdir(parents=True) - - # copy license to the constructor directory - shutil.copy(license_file, constructor_dir / "LICENSE") - - # write the post_install scripts referenced in the construct dict - if platform.startswith("win"): - with (constructor_dir / "post_install.bat").open("w") as f: - f.write("\n".join((r"del /q %PREFIX%\pkgs\*.tar.bz2", "exit 0", ""))) - else: - with (constructor_dir / "post_install.sh").open("w") as f: - f.write( - "\n".join( - ( - "#!/bin/sh", - f'PREFIX="${{PREFIX:-$2/{installer_name}}}"', - r"rm -f $PREFIX/pkgs/*.tar.bz2", - "exit 0", - "", - ) - ) - ) - - construct_yaml_path = constructor_dir / "construct.yaml" - with construct_yaml_path.open("w") as f: - yaml.safe_dump(construct_dict, stream=f) - - lockfile_contents = [ - f"# platform: {rendered_full_lock_spec.platform}", - f"# env_hash: {rendered_full_lock_spec.env_hash()}", - ] - lockfile_contents.extend(rendered_full_lock_spec.specs) - lock_file_path = constructor_dir / f"{constructor_name}.txt" - with lock_file_path.open("w") as f: - f.write("\n".join(lockfile_contents)) - - return constructor_specs + return rendered_platforms if __name__ == "__main__": @@ -221,8 +292,9 @@ if __name__ == "__main__": conda_executable=args.conda_exe, mamba=True, micromamba=True ) - constructor_specs = render_constructor_specs( + constructor_specs = render_platforms( environment_file=args.environment_file, + installer_pkg_specs=["mamba"], version=args.version, company=args.company, license_file=args.license_file,