367 lines
19 KiB
Python
367 lines
19 KiB
Python
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
from typer.testing import CliRunner
|
|
from worker.cli import app
|
|
from worker.config import load_config
|
|
from worker.gitea import Gitea
|
|
from worker.manifest import inspect_project, metadata_issues, repair_metadata
|
|
from worker.models import RustProject, BuildTarget
|
|
from worker.package import package, checksum
|
|
from worker.policy import archive_name, binstall_metadata, expected_tag
|
|
from worker.project import parse_repository, acquire_project, validate_checkout_tag
|
|
from worker.runner import run, CommandError
|
|
from worker.verify import verify_archive, verify_published, verify_binstall
|
|
|
|
|
|
class Fixture(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.manifest = self.root / 'Cargo.toml'
|
|
self.manifest.write_text('# Keep this comment\n[package]\nname = "demo"\nversion = "0.1.0"\n# Another comment\n')
|
|
self.project = RustProject(self.root, self.manifest, 'demo', '0.1.0',
|
|
'https://host/gitea/org/demo', ['tool'], 'a' * 40)
|
|
|
|
def artifacts(self):
|
|
binary = self.root / 'tool'
|
|
binary.write_bytes(b'example binary\x00')
|
|
binary.chmod(0o755)
|
|
return package(self.project, {'x86_64-unknown-linux-musl': binary,
|
|
'x86_64-pc-windows-gnu': binary})
|
|
|
|
|
|
class PolicyTests(Fixture):
|
|
def test_repository_urls(self):
|
|
for url in ('https://host/gitea/org/demo.git', 'https://host/gitea/org/demo/',
|
|
'git@host:gitea/org/demo.git', 'ssh://git@host/gitea/org/demo.git'):
|
|
repo = parse_repository(url)
|
|
self.assertEqual(repo.url, 'https://host/gitea/org/demo')
|
|
self.assertEqual(repo.base_url, 'https://host/gitea')
|
|
self.assertEqual(parse_repository('https://host:3000/org/demo').base_url, 'https://host:3000')
|
|
for bad in ('--upload-pack=evil', 'file:///tmp/repo', 'https://token@host/org/repo',
|
|
'https://host/org/repo?token=secret', 'https://host/../repo', 'https://host/org'):
|
|
with self.subTest(bad=bad), self.assertRaises(ValueError):
|
|
parse_repository(bad)
|
|
|
|
def test_naming(self):
|
|
self.assertEqual(expected_tag('0.1.0'), 'v0.1.0')
|
|
self.assertEqual(expected_tag('0.1.0', 'release-'), 'release-0.1.0')
|
|
self.assertEqual(archive_name('demo', 'target', '0.1.0'), 'demo-target-v0.1.0.tgz')
|
|
self.assertEqual(BuildTarget('x86_64-pc-windows-gnu').binary_filename('tool'), 'tool.exe')
|
|
self.assertEqual(BuildTarget('aarch64-unknown-linux-musl').binary_filename('tool'), 'tool')
|
|
self.assertIn('/release-{ version }/', binstall_metadata('release-')['pkg-url'])
|
|
self.assertEqual(binstall_metadata()['bin-dir'], '{ name }-{ target }-v{ version }/{ bin }{ binary-ext }')
|
|
|
|
def test_manifest_repair_preserves_comments_and_is_idempotent(self):
|
|
issues = metadata_issues(self.manifest, self.project.repository)
|
|
self.assertEqual(len(issues), 4)
|
|
repair_metadata(self.manifest, issues)
|
|
first = self.manifest.read_text()
|
|
self.assertIn('# Keep this comment', first)
|
|
self.assertIn('# Another comment', first)
|
|
self.assertEqual(metadata_issues(self.manifest, self.project.repository), [])
|
|
repair_metadata(self.manifest, [])
|
|
self.assertEqual(self.manifest.read_text(), first)
|
|
|
|
def test_conflicting_overrides_removed(self):
|
|
self.manifest.write_text(self.manifest.read_text() + '\n[package.metadata.binstall.overrides.x86_64-pc-windows-gnu]\npkg-fmt="zip"\n')
|
|
issues = metadata_issues(self.manifest, self.project.repository)
|
|
self.assertIn('package.metadata.binstall.overrides', [i.key for i in issues])
|
|
repair_metadata(self.manifest, issues)
|
|
self.assertEqual(metadata_issues(self.manifest, self.project.repository), [])
|
|
|
|
def test_workspace_and_multiple_binaries_rejected(self):
|
|
data = {'packages': [{'id': 'demo', 'name': 'demo', 'targets': [{'kind': ['bin'], 'name': 'one'},
|
|
{'kind': ['bin'], 'name': 'two'}]}],
|
|
'workspace_members': ['demo']}
|
|
with patch('worker.manifest.run', return_value=json.dumps(data)), self.assertRaisesRegex(ValueError, 'one, two'):
|
|
inspect_project(self.root)
|
|
self.manifest.write_text('[workspace]\nmembers=[]\n')
|
|
with patch('worker.manifest.run', return_value=json.dumps(data)), self.assertRaisesRegex(ValueError, '--package'):
|
|
inspect_project(self.root)
|
|
|
|
def test_symlink_manifest_cannot_repair_unrelated_file(self):
|
|
other = self.root / 'other.toml'
|
|
original = self.manifest.read_bytes()
|
|
self.manifest.rename(other)
|
|
self.manifest.symlink_to(other)
|
|
with self.assertRaisesRegex(ValueError, 'symlinked'):
|
|
repair_metadata(self.manifest, metadata_issues(self.manifest, self.project.repository))
|
|
self.assertEqual(other.read_bytes(), original)
|
|
|
|
def test_cross_metadata_hooks_rejected(self):
|
|
from worker.build import validate_cross_config
|
|
self.manifest.write_text(self.manifest.read_text() + '\n[workspace.metadata.cross.target.x86_64-unknown-linux-musl]\npre-build=["echo unsafe"]\n')
|
|
with self.assertRaisesRegex(ValueError, 'executable option'):
|
|
validate_cross_config(self.root)
|
|
|
|
def test_config_overrides_and_validation(self):
|
|
config = load_config(targets=['x86_64-pc-windows-gnu'], locked=False, profile='dev')
|
|
self.assertFalse(config.locked)
|
|
self.assertEqual(config.profile, 'dev')
|
|
for targets in ([], ['../../oops'], ['--target']):
|
|
with self.assertRaises(ValueError):
|
|
load_config(targets=targets)
|
|
|
|
def test_reproducible_archives_and_checksums(self):
|
|
artifacts, sums = self.artifacts()
|
|
again, _ = self.artifacts()
|
|
self.assertEqual([a.sha256 for a in artifacts], [a.sha256 for a in again])
|
|
for artifact in artifacts:
|
|
verify_archive(artifact.path, artifact)
|
|
self.assertEqual(artifact.sha256, hashlib.sha256(artifact.path.read_bytes()).hexdigest())
|
|
self.assertIn(f'{artifact.sha256} {artifact.path.name}\n', sums.read_text())
|
|
self.assertEqual(len(list(sums.parent.iterdir())), 3)
|
|
|
|
def test_malformed_archive_rejected_without_extraction(self):
|
|
artifacts, _ = self.artifacts()
|
|
path = self.root / 'malformed.tgz'
|
|
with tarfile.open(path, 'w:gz') as archive:
|
|
entry = tarfile.TarInfo('../../outside')
|
|
entry.size = 1
|
|
archive.addfile(entry, io.BytesIO(b'x'))
|
|
with self.assertRaises(ValueError):
|
|
verify_archive(path, artifacts[0])
|
|
self.assertFalse((self.root.parent / 'outside').exists())
|
|
|
|
def test_published_bytes_and_checksum_file(self):
|
|
artifacts, sums = self.artifacts()
|
|
paths = {p.name: p.read_bytes() for p in [a.path for a in artifacts] + [sums]}
|
|
def handler(request):
|
|
self.assertNotIn('authorization', request.headers)
|
|
return httpx.Response(200, content=paths[request.url.path.rsplit('/', 1)[1]])
|
|
verify_published(self.project.repository, 'v0.1.0', artifacts, sums, httpx.MockTransport(handler))
|
|
with self.assertRaisesRegex(ValueError, 'SHA-256'):
|
|
verify_published(self.project.repository, 'v0.1.0', artifacts, sums,
|
|
httpx.MockTransport(lambda r: httpx.Response(200, content=b'bad')))
|
|
|
|
def test_runner_failure_and_secret_redaction(self):
|
|
with patch.dict(os.environ, {'GITEA_TOKEN': 'super-secret'}):
|
|
with self.assertRaises(CommandError) as failure:
|
|
run('python3', '-c', 'import sys; print("super-secret", file=sys.stderr); sys.exit(3)')
|
|
self.assertNotIn('super-secret', str(failure.exception))
|
|
self.assertIn('(3)', str(failure.exception))
|
|
self.assertIn('[REDACTED]', str(failure.exception))
|
|
self.assertEqual(run('python3', '-c', 'import os; print(os.getenv("GITEA_TOKEN", "missing"))'), 'missing')
|
|
|
|
|
|
class GiteaTests(Fixture):
|
|
def client(self, handler):
|
|
env = patch.dict(os.environ, {'GITEA_TOKEN': 'secret-token'})
|
|
env.start()
|
|
self.addCleanup(env.stop)
|
|
client = Gitea(parse_repository(self.project.repository), 'GITEA_TOKEN', httpx.MockTransport(handler))
|
|
self.addCleanup(client.client.close)
|
|
return client
|
|
|
|
def test_create_and_reuse_release(self):
|
|
for exists in (False, True):
|
|
calls = []
|
|
def handler(request):
|
|
calls.append(request)
|
|
self.assertTrue(request.url.path.startswith('/gitea/api/v1/repos/org/demo/'))
|
|
if '/tags/v0.1.0' in request.url.path and '/releases/' not in request.url.path:
|
|
return httpx.Response(200, json={'commit': {'sha': self.project.commit}})
|
|
if request.method == 'POST':
|
|
body = json.loads(request.content)
|
|
self.assertTrue(body['prerelease'])
|
|
self.assertEqual(body['tag_name'], 'v0.1.0')
|
|
return httpx.Response(201, json={'id': 1})
|
|
return httpx.Response(200 if exists else 404, json={'id': 1})
|
|
self.assertEqual(self.client(handler).ensure_release('v0.1.0', self.project.commit, prerelease=True)['id'], 1)
|
|
self.assertEqual(sum(r.method == 'POST' for r in calls), int(not exists))
|
|
|
|
def test_collision_preflight_and_replace(self):
|
|
paths = [self.root / 'one.tgz', self.root / 'two.tgz']
|
|
for path in paths:
|
|
path.write_bytes(b'archive')
|
|
for replace in (False, True):
|
|
calls = []
|
|
def handler(request):
|
|
calls.append(request.method)
|
|
if request.method == 'GET':
|
|
return httpx.Response(200, json=[{'id': 9, 'name': 'two.tgz'}])
|
|
return httpx.Response(204) if request.method == 'DELETE' else httpx.Response(201, json={'id': 10})
|
|
client = self.client(handler)
|
|
if replace:
|
|
client.upload(1, paths, True)
|
|
self.assertEqual(calls, ['GET', 'POST', 'DELETE', 'POST'])
|
|
else:
|
|
with self.assertRaisesRegex(ValueError, 'already exist'):
|
|
client.upload(1, paths)
|
|
self.assertEqual(calls, ['GET'])
|
|
|
|
def test_auth_failure_does_not_echo_body(self):
|
|
client = self.client(lambda r: httpx.Response(401, text='secret-token'))
|
|
with self.assertRaises(RuntimeError) as failure:
|
|
client.ensure_release('v0.1.0', self.project.commit)
|
|
self.assertNotIn('secret-token', str(failure.exception))
|
|
self.assertIn('401', str(failure.exception))
|
|
|
|
def test_wrong_remote_commit_blocks_creation(self):
|
|
client = self.client(lambda r: httpx.Response(200, json={'commit': {'sha': 'wrong'}}))
|
|
with self.assertRaisesRegex(ValueError, 'differs'):
|
|
client.ensure_release('v0.1.0', self.project.commit)
|
|
|
|
|
|
class CliTests(Fixture):
|
|
def test_dry_run_never_repairs_or_builds(self):
|
|
original = self.manifest.read_bytes()
|
|
with patch('worker.cli.inspect_project', return_value=self.project), \
|
|
patch('worker.cli.prerequisites', return_value=([], 'podman')), \
|
|
patch('worker.cli.validate_checkout_tag', return_value=True), \
|
|
patch('worker.cli.build') as build, patch('worker.cli.Gitea') as api:
|
|
result = CliRunner().invoke(app, [str(self.root), '--dry-run', '--fix-metadata'])
|
|
self.assertEqual(result.exit_code, 1, result.output)
|
|
self.assertIn('Dry run complete', result.output)
|
|
self.assertEqual(self.manifest.read_bytes(), original)
|
|
build.assert_not_called()
|
|
api.assert_not_called()
|
|
|
|
def test_stop_metadata_and_tag_mismatch(self):
|
|
with patch('worker.cli.inspect_project', return_value=self.project), patch('worker.cli.prerequisites') as prereq:
|
|
result = CliRunner().invoke(app, [str(self.root), '--fix-metadata', '--stop-after', 'metadata'])
|
|
self.assertEqual(result.exit_code, 0, result.output)
|
|
prereq.assert_not_called()
|
|
with patch('worker.cli.inspect_project', return_value=self.project):
|
|
result = CliRunner().invoke(app, [str(self.root), '--tag', 'v9.9.9'])
|
|
self.assertNotEqual(result.exit_code, 0)
|
|
|
|
def test_stop_package_never_contacts_gitea(self):
|
|
repair_metadata(self.manifest, metadata_issues(self.manifest, self.project.repository))
|
|
binary = self.root / 'binary'
|
|
binary.write_bytes(b'fake')
|
|
with patch('worker.cli.inspect_project', return_value=self.project), \
|
|
patch('worker.cli.prerequisites', return_value=([], 'podman')), \
|
|
patch('worker.cli.validate_checkout_tag', return_value=False), \
|
|
patch('worker.cli.verify_source'), \
|
|
patch('worker.cli.build', return_value={'x86_64-unknown-linux-musl': binary}), \
|
|
patch('worker.cli.Gitea') as api:
|
|
result = CliRunner().invoke(app, [str(self.root), '--stop-after', 'package'])
|
|
self.assertEqual(result.exit_code, 0, result.output)
|
|
api.assert_not_called()
|
|
self.assertEqual(len(list((self.root / 'dist').glob('*/SHA256SUMS'))), 1)
|
|
|
|
def test_local_source_never_removed(self):
|
|
with acquire_project(str(self.root)) as root:
|
|
self.assertEqual(root, self.root)
|
|
self.assertTrue(self.manifest.exists())
|
|
|
|
|
|
class WrapperTests(Fixture):
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.wrapper = self.root / 'release.sh'
|
|
shutil.copy(Path(__file__).resolve().parents[1] / 'release.sh', self.wrapper)
|
|
|
|
def invoke(self, env):
|
|
clean = os.environ.copy()
|
|
clean.pop('VIRTUAL_ENV', None)
|
|
clean.update(env)
|
|
return subprocess.run(['bash', str(self.wrapper), '--help'], stdin=subprocess.DEVNULL,
|
|
capture_output=True, text=True, env=clean, cwd='/tmp')
|
|
|
|
def test_no_venv_noninteractive(self):
|
|
result = self.invoke({})
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn('non-interactive', result.stderr)
|
|
|
|
def test_existing_venv_needs_activation(self):
|
|
(self.root / '.venv').mkdir()
|
|
result = self.invoke({})
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn(str(self.root / '.venv/bin/activate'), result.stdout)
|
|
|
|
def test_wrong_venv_even_without_project_venv(self):
|
|
result = self.invoke({'VIRTUAL_ENV': '/tmp'})
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn('non-interactive', result.stderr)
|
|
|
|
def test_correct_venv_preserves_cwd(self):
|
|
bin_dir = self.root / '.venv' / 'bin'
|
|
bin_dir.mkdir(parents=True)
|
|
interpreter = bin_dir / 'python'
|
|
interpreter.write_text('#!/bin/sh\npwd\nprintf "%s\\n" "$@"\n')
|
|
interpreter.chmod(0o755)
|
|
result = self.invoke({'VIRTUAL_ENV': str(bin_dir.parent)})
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(result.stdout.splitlines(), ['/tmp', '-P', '-m', 'worker', '--help'])
|
|
|
|
|
|
class PipelineTests(Fixture):
|
|
def test_full_pipeline_and_every_stage_boundary(self):
|
|
from contextlib import ExitStack
|
|
stages = ['inspect', 'metadata', 'verify', 'build', 'package', 'release', 'upload', 'binstall']
|
|
repair_metadata(self.manifest, metadata_issues(self.manifest, self.project.repository))
|
|
binary = self.root / 'tool'
|
|
binary.write_bytes(b'release binary')
|
|
for index, stage in enumerate(stages):
|
|
calls = []
|
|
with self.subTest(stage=stage), ExitStack() as stack:
|
|
stack.enter_context(patch.dict(os.environ, {'GITEA_TOKEN': 'test-token'}))
|
|
stack.enter_context(patch('worker.cli.inspect_project', return_value=self.project))
|
|
stack.enter_context(patch('worker.cli.runner.run', return_value=''))
|
|
stack.enter_context(patch('worker.cli.validate_checkout_tag', return_value=True))
|
|
stack.enter_context(patch('worker.cli.prerequisites', return_value=([], 'podman')))
|
|
stack.enter_context(patch('worker.cli.verify_source', side_effect=lambda *a: calls.append('verify')))
|
|
def fake_build(*args):
|
|
calls.append('build')
|
|
return {'x86_64-unknown-linux-musl': binary}
|
|
stack.enter_context(patch('worker.cli.build', side_effect=fake_build))
|
|
def fake_package(*args):
|
|
calls.append('package')
|
|
return package(*args)
|
|
stack.enter_context(patch('worker.cli.package', side_effect=fake_package))
|
|
api = stack.enter_context(patch('worker.cli.Gitea')).return_value.__enter__.return_value
|
|
def fake_release(*args):
|
|
calls.append('release')
|
|
return {'id': 1, 'draft': False}
|
|
api.ensure_release.side_effect = fake_release
|
|
api.upload.side_effect = lambda *a: calls.append('upload')
|
|
stack.enter_context(patch('worker.cli.verify_published', side_effect=lambda *a: calls.append('download')))
|
|
stack.enter_context(patch('worker.cli.verify_binstall', side_effect=lambda *a: calls.append('binstall')))
|
|
result = CliRunner().invoke(app, [str(self.root), '--stop-after', stage])
|
|
self.assertEqual(result.exit_code, 0, result.output)
|
|
expected = stages[2:index + 1]
|
|
if stage == 'binstall':
|
|
expected = expected[:-1] + ['download', 'binstall']
|
|
self.assertEqual(calls, expected)
|
|
|
|
def test_git_local_tag_policy_and_clone_cleanup(self):
|
|
run('git', 'init', str(self.root))
|
|
run('git', 'add', 'Cargo.toml', cwd=self.root)
|
|
run('git', '-c', 'commit.gpgsign=false', '-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-m', 'fixture', cwd=self.root)
|
|
run('git', '-c', 'tag.gpgsign=false', 'tag', 'v0.1.0', cwd=self.root)
|
|
self.assertTrue(validate_checkout_tag(self.root, 'v0.1.0'))
|
|
self.assertFalse(validate_checkout_tag(self.root, 'v0.2.0', required=False))
|
|
self.manifest.write_text(self.manifest.read_text() + '\n# newer commit\n')
|
|
run('git', 'add', 'Cargo.toml', cwd=self.root)
|
|
run('git', '-c', 'commit.gpgsign=false', '-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-m', 'second', cwd=self.root)
|
|
with self.assertRaisesRegex(ValueError, 'does not point'):
|
|
validate_checkout_tag(self.root, 'v0.1.0')
|
|
managed = self.root / 'managed'
|
|
managed.mkdir()
|
|
with patch('worker.project.tempfile.mkdtemp', return_value=str(managed)), \
|
|
patch('worker.project.run', side_effect=CommandError('clone failed')):
|
|
with self.assertRaises(CommandError):
|
|
with acquire_project('https://host/org/demo'):
|
|
pass
|
|
self.assertFalse(managed.exists())
|
|
self.assertTrue(self.manifest.exists())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|