diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 75c0b37..031abc3 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -150,7 +150,7 @@ jobs: with: artichoke_ref: ${{ steps.release_info.outputs.commit }} target_triple: ${{ matrix.target }} - output_file: ${{ github.workspace }}/THIRDPARTY + output_file: ${{ github.workspace }}/THIRDPARTY.txt - name: Clone Artichoke uses: actions/checkout@v3 @@ -208,14 +208,65 @@ jobs: working-directory: artichoke run: cargo build --verbose --release --target ${{ matrix.target }} + # This will codesign binaries in place which means that the tarballed + # binaries will be codesigned as well. + - name: Run Apple Codesigning and Notarization + id: apple_codesigning + if: runner.os == 'macOS' + run: | + python3 macos_sign_and_notarize.py "artichoke-nightly-${{ matrix.target }}" \ + --binary "artichoke/target/${{ matrix.target }}/release/artichoke" \ + --binary "artichoke/target/${{ matrix.target }}/release/airb" \ + --resource artichoke/LICENSE \ + --resource artichoke/README.md \ + --resource THIRDPARTY.txt + env: + MACOS_NOTARIZE_APP_PASSWORD: ${{ secrets.MACOS_NOTARIZE_APP_PASSWORD }} + MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} + MACOS_CERTIFICATE_PASSPHRASE: ${{ secrets.MACOS_CERTIFICATE_PASSPHRASE }} + + - name: GPG sign Apple DMG + id: apple_codesigning_gpg + if: runner.os == 'macOS' + run: | + python3 gpg_sign.py "artichoke-nightly-${{ matrix.target }}" \ + --artifact "${{ steps.apple_codesigning.outputs.asset }}" + + - name: Upload release archive + uses: ncipollo/release-action@v1 + if: runner.os == 'macOS' + with: + token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ steps.release_info.outputs.version }} + draft: true + allowUpdates: true + omitBodyDuringUpdate: true + omitNameDuringUpdate: true + omitPrereleaseDuringUpdate: true + artifacts: ${{ steps.apple_codesigning.outputs.asset }} + artifactContentType: ${{ steps.apple_codesigning.outputs.content_type }} + + - name: Upload release signature + uses: ncipollo/release-action@v1 + if: runner.os == 'macOS' + with: + token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ steps.release_info.outputs.version }} + draft: true + allowUpdates: true + omitBodyDuringUpdate: true + omitNameDuringUpdate: true + omitPrereleaseDuringUpdate: true + artifacts: ${{ steps.apple_codesigning_gpg.outputs.signature }} + artifactContentType: "text/plain" + - name: Build archive shell: bash id: build run: | staging="artichoke-nightly-${{ matrix.target }}" mkdir -p "$staging"/ - cp artichoke/{README.md,LICENSE} "$staging/" - cp THIRDPARTY "$staging/THIRDPARTY.txt" + cp artichoke/{README.md,LICENSE} THIRDPARTY.txt "$staging/" if [ "${{ runner.os }}" = "Windows" ]; then cp "artichoke/target/${{ matrix.target }}/release/artichoke.exe" "$staging/" cp "artichoke/target/${{ matrix.target }}/release/airb.exe" "$staging/" diff --git a/.gitignore b/.gitignore index 2b8c6cf..d7cee9e 100644 --- a/.gitignore +++ b/.gitignore @@ -225,3 +225,4 @@ build-iPhoneSimulator/ *.dmg /dist/* +*.keychain-db diff --git a/apple-certs/DeveloperIDG2CA.cer b/apple-certs/DeveloperIDG2CA.cer new file mode 100644 index 0000000..8cbcf6f Binary files /dev/null and b/apple-certs/DeveloperIDG2CA.cer differ diff --git a/apple-certs/README.md b/apple-certs/README.md new file mode 100644 index 0000000..5dcebc7 --- /dev/null +++ b/apple-certs/README.md @@ -0,0 +1,22 @@ +# Apple Certificates + +Several certificates are required by the build keychain used in the codesigning +process. + +## Certificate Chain + +The build keychain must include the all intermediate certificates for the +codesigning certificate. + +All of Apple's CAs can be found at: +. + +The Developer ID Application certificate used for codesigning has "Developer +ID - G2 (Expiring 09/17/2031 00:00:00 UTC)" as an intermediate in its +certificate chain. + +## Provisioning Profile + +`artichoke-provisioning-profile-signing.cer` contains a provisioning profile +which is associated with the Developer ID application and is required for +signing. diff --git a/apple-certs/artichoke-provisioning-profile-signing.cer b/apple-certs/artichoke-provisioning-profile-signing.cer new file mode 100644 index 0000000..bb3d29e Binary files /dev/null and b/apple-certs/artichoke-provisioning-profile-signing.cer differ diff --git a/gpg_sign.py b/gpg_sign.py index 60dc073..ea0459c 100755 --- a/gpg_sign.py +++ b/gpg_sign.py @@ -92,6 +92,7 @@ def gpg_sign_artifact(*, artifact_path, release_name): str(asc), str(artifact_path), ], + check=True, # capture output because `gpg --detatch-sign` writes to stderr which # prevents the GitHub Actions log group from working correctly. stdout=subprocess.PIPE, @@ -120,6 +121,7 @@ def validate(*, artifact_name, asc): str(asc), str(artifact_name), ], + check=True, # capture output because `gpg --verify` writes to stderr which # prevents the GitHub Actions log group from working correctly. stdout=subprocess.PIPE, @@ -160,7 +162,7 @@ def main(args): for artifact in artifacts: if not artifact.is_file(): - print("Error: {artifact} does not exist", file=sys.stderr) + print(f"Error: artifact file {artifact} does not exist", file=sys.stderr) return 1 if len(artifacts) > 1: @@ -185,16 +187,25 @@ def main(args): return 0 except subprocess.CalledProcessError as e: - print( - f"""Error: failed to invoke command. - \tCommand: {e.cmd} - \tReturn Code: {e.returncode}""", - file=sys.stderr, - ) + print("Error: failed to invoke command", file=sys.stderr) + print(f" Command: {e.cmd}", file=sys.stderr) + print(f" Return Code: {e.returncode}", file=sys.stderr) + if e.stdout: + print() + print("Output:", file=sys.stderr) + for line in e.stdout.splitlines(): + print(f" {line}", file=sys.stderr) + if e.stderr: + print() + print("Error Output:", file=sys.stderr) + for line in e.stderr.splitlines(): + print(f" {line}", file=sys.stderr) + print() + print(traceback.format_exc(), file=sys.stderr) return e.returncode except Exception as e: print(f"Error: {e}", file=sys.stderr) - print(traceback.format_exc()) + print(traceback.format_exc(), file=sys.stderr) return 1 diff --git a/macos_sign_and_notarize.py b/macos_sign_and_notarize.py index 418c1ac..2606b19 100755 --- a/macos_sign_and_notarize.py +++ b/macos_sign_and_notarize.py @@ -15,6 +15,28 @@ from pathlib import Path +def run_command_with_merged_output(command): + """ + Run the given command as a subprocess and merge its stdout and stderr + streams. + + This is useful for funnelling all output of a command into a GitHub Actions + log group. + + This command uses `check=True` when delegating to `subprocess`. + """ + + proc = subprocess.run( + command, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + for line in proc.stdout.splitlines(): + print(line) + + def set_output(*, name, value): """ Set an output for a GitHub Actions job. @@ -55,15 +77,14 @@ def attach_disk_image(image, *, readwrite=False): ] else: command = ["/usr/bin/hdiutil", "attach", str(image)] + run_command_with_merged_output(command) - subprocess.run(command, check=True) mounted_image = disk_image_mount_path() yield mounted_image finally: with log_group("Detatching disk image"): - subprocess.run( + run_command_with_merged_output( ["/usr/bin/hdiutil", "detach", str(mounted_image)], - check=True, ) @@ -98,7 +119,7 @@ def get_image_size(image): capture_output=True, text=True, ) - size = int(proc.stdout.split(" ")[0]) + size = int(proc.stdout.split()[0]) return (size * 512 / 1000 / 1000) + 1 @@ -176,9 +197,9 @@ def notarization_app_specific_password(): codesigning identity's Apple ID. """ - if app_specific_password := os.getenv("APPLE_ID_APP_PASSWORD"): + if app_specific_password := os.getenv("MACOS_NOTARIZE_APP_PASSWORD"): return app_specific_password - raise Exception("APPLE_ID_APP_PASSWORD environment variable is required") + raise Exception("MACOS_NOTARIZE_APP_PASSWORD environment variable is required") def notarization_team_id(): @@ -218,32 +239,60 @@ def create_keychain(*, keychain_password): with log_group("Setup notarization keychain"): # security create-keychain -p "$keychain_password" "$keychain_path" - subprocess.run( + run_command_with_merged_output( [ "security", "create-keychain", "-p", keychain_password, str(keychain_path()), - ], - check=True, + ] ) + print(f"Created keychain at {keychain_path()}") + # security set-keychain-settings -lut 900 "$keychain_path" - subprocess.run( - ["security", "set-keychain-settings", "-lut", "900", str(keychain_path())], - check=True, + run_command_with_merged_output( + ["security", "set-keychain-settings", "-lut", "900", str(keychain_path())] ) + print("Set keychain to be ephemeral") + # security unlock-keychain -p "$keychain_password" "$keychain_path" - subprocess.run( + run_command_with_merged_output( [ "security", "unlock-keychain", "-p", keychain_password, str(keychain_path()), - ], + ] + ) + print(f"Unlocked keychain at {keychain_path()}") + + # Per `man codesign`, the keychain filename passed via the `--keychain` + # argument will not be searched to resolve the signing identity's + # certificate chain unless it is also on the user's keychain search list. + # + # `security create-keychain` does not add keychains to the search path. + # _Opening_ them does, as well as explicitly manipulating the search path + # with `security list-keychains -s`. + # + # This stackoverflow post explains the solution: + # + # + # `security delete-keychain` removes the keychain from the search path. + proc = subprocess.run( + ["security", "list-keychains", "-d", "user"], check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + search_path = [line.strip().strip('"') for line in proc.stdout.splitlines()] + search_path.append(str(keychain_path())) + run_command_with_merged_output( + ["security", "list-keychains", "-d", "user", "-s"] + search_path ) + print(f"Set keychain search path: {', '.join(search_path)}") def delete_keychain(): @@ -256,14 +305,19 @@ def delete_keychain(): """ with log_group("Delete keychain"): - try: - # security delete-keychain /path/to/notarization.keychain-db - subprocess.run( - ["security", "delete-keychain", str(keychain_path())], - check=True, - ) + # security delete-keychain /path/to/notarization.keychain-db + proc = subprocess.run( + ["security", "delete-keychain", str(keychain_path())], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + for line in proc.stdout.splitlines(): + print(line) + + if proc.returncode == 0: print(f"Keychain deleted from {keychain_path()}") - except subprocess.CalledProcessError: + else: # keychain does not exist print(f"Keychain not found at {keychain_path()}, ignoring ...") @@ -281,10 +335,10 @@ def import_notarization_credentials(): # xcrun notarytool store-credentials \ # "$notarytool_credentials_profile" \ # --apple-id "apple-codesign@artichokeruby.org" \ - # --password "$APPLE_ID_APP_PASSWORD" \ + # --password "$MACOS_NOTARIZE_APP_PASSWORD" \ # --team-id "VDKP67932G" \ # --keychain "$keychain_path" - subprocess.run( + run_command_with_merged_output( [ "/usr/bin/xcrun", "notarytool", @@ -299,10 +353,36 @@ def import_notarization_credentials(): "--keychain", str(keychain_path()), ], - check=True, ) +def import_certificate(*, path, name=None, password=None): + """ + Import a certificate at a given path into the build keychain. + """ + + # security import certificate.p12 \ + # -k "$keychain_path" \ + # -P "$MACOS_CERTIFICATE_PWD" \ + # -T /usr/bin/codesign + command = [ + "security", + "import", + str(path), + "-k", + str(keychain_path()), + "-T", + "/usr/bin/codesign", + ] + if password is not None: + command.extend(["-P", password]) + + run_command_with_merged_output(command) + + cert_name = path if name is None else name + print(f"Imported certificate {cert_name}") + + def import_codesigning_certificate(): """ Import codesigning certificate into the codesigning and notarization process @@ -326,34 +406,32 @@ def import_codesigning_certificate(): except binascii.Error: raise Exception("MACOS_CERTIFICATE must be base64 encoded") - certificate_password = os.getenv("MACOS_CERTIFICATE_PWD") + certificate_password = os.getenv("MACOS_CERTIFICATE_PASSPHRASE") if not certificate_password: raise Exception( - "MACOS_CERTIFICATE_PASSWORD environment variable is required" + "MACOS_CERTIFICATE_PASSPHRASE environment variable is required" ) with tempfile.TemporaryDirectory() as tempdirname: cert = Path(tempdirname).joinpath("certificate.p12") cert.write_bytes(certificate) - # security import certificate.p12 \ - # -k "$keychain_path" \ - # -P "$MACOS_CERTIFICATE_PWD" \ - # -T /usr/bin/codesign - subprocess.run( - [ - "security", - "import", - str(cert), - "-k", - str(keychain_path()), - "-P", - certificate_password, - "-T", - "/usr/bin/codesign", - ], - check=True, + import_certificate( + path=cert, name="Developer Application", password=certificate_password ) + with log_group("Import provisioning profile"): + import_certificate( + path="apple-certs/artichoke-provisioning-profile-signing.cer" + ) + + with log_group("Import certificate chain"): + import_certificate(path="apple-certs/DeveloperIDG2CA.cer") + + with log_group("Show codesigning identities"): + run_command_with_merged_output( + ["security", "find-identity", "-p", "codesigning", str(keychain_path())] + ) + def setup_codesigning_and_notarization_keychain(*, keychain_password): """ @@ -372,7 +450,7 @@ def setup_codesigning_and_notarization_keychain(*, keychain_password): # security set-key-partition-list \ # -S "apple-tool:,apple:,codesign:" \ # -s -k "$keychain_password" "$keychain_path" - subprocess.run( + run_command_with_merged_output( [ "security", "set-key-partition-list", @@ -382,8 +460,7 @@ def setup_codesigning_and_notarization_keychain(*, keychain_password): "-k", keychain_password, str(keychain_path()), - ], - check=True, + ] ) @@ -402,7 +479,7 @@ def codesign_binary(*, binary_path): # --force \ # "$binary_path" with log_group(f"Run codesigning [{binary_path.name}]"): - subprocess.run( + run_command_with_merged_output( [ "/usr/bin/codesign", "--keychain", @@ -418,8 +495,7 @@ def codesign_binary(*, binary_path): "-vvv", "--force", str(binary_path), - ], - check=True, + ] ) @@ -465,7 +541,7 @@ def create_notarization_bundle(*, release_name, binaries, resources): # -volname "Artichoke Ruby nightly" \ # -srcfolder "$release_name" \ # -ov -format UDRW name.dmg - subprocess.run( + run_command_with_merged_output( [ "/usr/bin/hdiutil", "create", @@ -479,8 +555,7 @@ def create_notarization_bundle(*, release_name, binaries, resources): "UDRW", "-verbose", str(dmg_writable), - ], - check=True, + ] ) with log_group("Set disk image icon"): @@ -492,40 +567,28 @@ def create_notarization_bundle(*, release_name, binaries, resources): "https://artichoke.github.io/logo/Artichoke-dmg.icns", str(icns) ) shutil.copy(icns, dmg_icns_path) - subprocess.run( - [ - "/usr/bin/SetFile", - "-c", - "icnC", - str(dmg_icns_path), - ], - check=True, + run_command_with_merged_output( + ["/usr/bin/SetFile", "-c", "icnC", str(dmg_icns_path)] ) + # Tell the volume that it has a special file attribute - subprocess.run( - [ - "/usr/bin/SetFile", - "-a", - "C", - str(mounted_image), - ], - check=True, + run_command_with_merged_output( + ["/usr/bin/SetFile", "-a", "C", str(mounted_image)] ) with log_group("Shrink disk image to fit"): - subprocess.run( + run_command_with_merged_output( [ "/usr/bin/hdiutil", "resize", "-size", f"{get_image_size(dmg_writable)}m", str(dmg_writable), - ], - check=True, + ] ) with log_group("Compress disk image"): - subprocess.run( + run_command_with_merged_output( [ "/usr/bin/hdiutil", "convert", @@ -536,9 +599,9 @@ def create_notarization_bundle(*, release_name, binaries, resources): "zlib-level=9", "-o", str(dmg), - ], - check=True, + ] ) + dmg_writable.unlink() codesign_binary(binary_path=dmg) @@ -620,15 +683,8 @@ def staple_bundle(*, bundle): """ with log_group("Staple disk image"): - subprocess.run( - [ - "/usr/bin/xcrun", - "stapler", - "staple", - "-v", - str(bundle), - ], - check=True, + run_command_with_merged_output( + ["/usr/bin/xcrun", "stapler", "staple", "-v", str(bundle)] ) @@ -638,15 +694,8 @@ def validate(*, bundle, binary_names): """ with log_group("Verify disk image staple"): - subprocess.run( - [ - "/usr/bin/xcrun", - "stapler", - "validate", - "-v", - str(bundle), - ], - check=True, + run_command_with_merged_output( + ["/usr/bin/xcrun", "stapler", "validate", "-v", str(bundle)] ) with log_group("Verify disk image signature"): @@ -654,7 +703,7 @@ def validate(*, bundle, binary_names): # --context context:primary-signature \ # 2022-09-03-test-codesign-notarize-dmg-v1.dmg \ # -v - subprocess.run( + run_command_with_merged_output( [ "/usr/sbin/spctl", "-a", @@ -664,15 +713,14 @@ def validate(*, bundle, binary_names): "context:primary-signature", str(bundle), "-v", - ], - check=True, + ] ) with attach_disk_image(bundle) as mounted_image: for binary in binary_names: mounted_binary = mounted_image.joinpath(binary) with log_group(f"Verify signature: {binary}"): - subprocess.run( + run_command_with_merged_output( [ "/usr/bin/codesign", "--verify", @@ -681,20 +729,18 @@ def validate(*, bundle, binary_names): "--strict=all", "-vvv", str(mounted_binary), - ], - check=True, + ] ) with log_group(f"Display signature: {binary}"): - subprocess.run( + run_command_with_merged_output( [ "/usr/bin/codesign", "--display", "--check-notarization", "-vvv", str(mounted_binary), - ], - check=True, + ] ) @@ -733,7 +779,11 @@ def main(args): for binary in binaries: if not binary.is_file(): - print("Error: {binary} does not exist", file=sys.stderr) + print(f"Error: binary file {binary} does not exist", file=sys.stderr) + return 1 + for resource in resources: + if not resource.is_file(): + print(f"Error: resource file {resource} does not exist", file=sys.stderr) return 1 try: @@ -754,20 +804,30 @@ def main(args): staple_bundle(bundle=bundle) validate(bundle=bundle, binary_names=[binary.name for binary in binaries]) - set_output(name="bundle", value=bundle) + set_output(name="asset", value=bundle) + set_output(name="content_type", value="application/x-apple-diskimage") return 0 except subprocess.CalledProcessError as e: - print( - f"""Error: failed to invoke command. - \tCommand: {e.cmd} - \tReturn Code: {e.returncode}""", - file=sys.stderr, - ) + print("Error: failed to invoke command", file=sys.stderr) + print(f" Command: {e.cmd}", file=sys.stderr) + print(f" Return Code: {e.returncode}", file=sys.stderr) + if e.stdout: + print() + print("Output:", file=sys.stderr) + for line in e.stdout.splitlines(): + print(f" {line}", file=sys.stderr) + if e.stderr: + print() + print("Error Output:", file=sys.stderr) + for line in e.stderr.splitlines(): + print(f" {line}", file=sys.stderr) + print() + print(traceback.format_exc(), file=sys.stderr) return e.returncode except Exception as e: print(f"Error: {e}", file=sys.stderr) - print(traceback.format_exc()) + print(traceback.format_exc(), file=sys.stderr) return 1 finally: # Purge keychain.