From a80eeafb9e16ad8a2c2876a523cc3bbe51028f9b Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Sat, 3 Oct 2026 22:37:33 +0200 Subject: [PATCH] Add Python CI quality and dependency gates --- .gitignore | 2 +- Jenkinsfile | 39 ++++++++++++++++++++++++--- README.md | 51 ++++++++++++++++++++++++++++++++---- fix_submodules.py | 13 +++++++-- pyproject.toml | 35 +++++++++++++++++++++++++ requirements-ci.txt | 4 +++ tests/test_fix_submodules.py | 40 +++++++++++++++++++++++++++- 7 files changed, 171 insertions(+), 13 deletions(-) create mode 100644 pyproject.toml create mode 100644 requirements-ci.txt diff --git a/.gitignore b/.gitignore index 5d381cc..d3c82b9 100644 --- a/.gitignore +++ b/.gitignore @@ -123,6 +123,7 @@ celerybeat.pid # Environments .env .venv +.venv-app/ env/ venv/ ENV/ @@ -159,4 +160,3 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ - diff --git a/Jenkinsfile b/Jenkinsfile index 86b5c16..a40f09c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,6 +16,7 @@ pipeline { environment { GIT_TERMINAL_PROMPT = '0' PYTHONUNBUFFERED = '1' + PIP_DISABLE_PIP_VERSION_CHECK = '1' } stages { @@ -29,9 +30,35 @@ pipeline { sh 'python3 --version && git --version' } } - stage('Unit tests') { + stage('Set up Python tools') { steps { - sh 'python3 -m unittest discover -s tests -v' + sh ''' + python3 -m venv .venv + .venv/bin/python -m pip install -r requirements-ci.txt + python3 -m venv .venv-app + .venv-app/bin/python -m pip install setuptools==80.9.0 + .venv-app/bin/python -m pip install --no-build-isolation --no-deps . + ''' + } + } + stage('Dependency vulnerability audit') { + steps { + sh '.venv/bin/pip-audit --strict .' + } + } + stage('Dependency license policy') { + steps { + sh '.venv/bin/pip-licenses --python=.venv-app/bin/python' + } + } + stage('Unit tests and coverage') { + steps { + sh ''' + .venv/bin/coverage erase + .venv/bin/coverage run -m unittest discover -s tests -v + .venv/bin/coverage report + .venv/bin/coverage xml + ''' } } stage('Ravenous clone test') { @@ -43,7 +70,11 @@ pipeline { } post { - success { echo 'repo-fix tests passed.' } - failure { echo 'repo-fix tests failed; see the test output above.' } + success { + echo 'repo-fix audit, license policy, coverage, and tests passed.' + } + failure { + echo 'repo-fix CI failed; check the audit, policy, coverage, and test stages above.' + } } } diff --git a/README.md b/README.md index 6e12299..d94df27 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Fix Dandaedre Gitea submodule URLs after cloning, pulling, or syncing: - `.git/modules/**/config` uses the same SSH format for submodule remotes (`url` and `pushurl`) and nested submodule URL entries. -Requires Python 3 and Git on your PATH. No pip packages are needed; the script +Requires Python 3.10 or newer and Git on your PATH. No runtime pip packages are needed; the script uses the Python standard library and Git's config commands. Preview the changes, then apply them: @@ -17,6 +17,12 @@ python3 fix_submodules.py /path/to/repository --dry-run python3 fix_submodules.py /path/to/repository ``` +The project follows semantic versioning and currently reports version `0.1.0`: + +```bash +python3 fix_submodules.py --version +``` + The repository defaults to the current directory, and may also be a directory inside a working tree. Run the script again whenever a pull or `git submodule sync` changes the URLs. Repeated runs are safe. Dry runs do not write files. @@ -51,16 +57,51 @@ Run tests: python3 -m unittest discover -s tests -v ``` +The unit suite includes a small deterministic fuzz test and malformed-value +cases for URL parsing. They are intended to catch crashes and unsafe path +shapes, not to replace a dedicated fuzzer. + +For the same coverage check as CI, install the pinned tools and run: + +```bash +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-ci.txt +.venv/bin/coverage run -m unittest discover -s tests -v +.venv/bin/coverage report +``` + +Branch coverage must remain at or above 80%. This deliberately modest floor +keeps the check useful without forcing low-value tests. + ## Jenkins test CI The `Jenkinsfile` uses the `rust_no_docker` agent from the Ravenous pipeline. Configure a Jenkins Pipeline from SCM (or a multibranch pipeline) for this repository with script path `Jenkinsfile`. The agent needs `python3` and `git` -on PATH and network access to the Gitea HTTPS proxy. No uv, Docker, or package -installation is required. Use the Gitea webhook URL from the Jenkinsfile if -your Jenkins Gitea integration is configured at that address. +on PATH and network access to PyPI, vulnerability data, and the Gitea HTTPS +proxy. No uv or Docker is required; CI creates isolated virtual environments +and installs the pinned tools. Use the Gitea webhook URL from the Jenkinsfile +if your Jenkins Gitea integration is configured at that address. -Every build runs the unit tests. By default, it also shallow-clones +Every build runs the unit tests with coverage. Before testing, it runs two +small supply-chain gates: `pip-audit` 2.10.1 fails on known vulnerabilities or +dependency-collection errors, and `pip-licenses` 5.5.5 allows only MIT-licensed +installed application packages. The CI environment also pins coverage 7.16.1 +and setuptools 80.9.0 in `requirements-ci.txt`. Run the gates locally with: + +```bash +.venv/bin/pip-audit --strict . +.venv/bin/pip-licenses --python=.venv-app/bin/python +``` + +The audit needs network access for package resolution and current advisory +data. The license gate inspects the isolated application environment; update +the explicit allow-list only after reviewing a new dependency. The project has +no runtime dependencies today. These checks do not analyze the standard +library, Git, malicious packages, provenance, or undisclosed vulnerabilities; +no broader hardening tools are in scope. + +By default, CI also shallow-clones `Ravenous/ravenous_roll20.git` through the HTTPS proxy into a temporary directory and verifies dry-run safety, both URL formats, repeatability, recovery after `git submodule sync`, and correction of the old SSH hostname. The clone is diff --git a/fix_submodules.py b/fix_submodules.py index 0d42db2..92fcae7 100644 --- a/fix_submodules.py +++ b/fix_submodules.py @@ -12,15 +12,23 @@ from urllib.parse import urlsplit HTTPS_BASE = "https://proxy.dandaedre.com/gitea/" SSH_BASE = "git@git.dandaedre.com:" HOSTS = {"dandaedre.com", "git.dandaedre.com", "proxy.dandaedre.com"} +__version__ = "0.1.0" def repository_path(url): """Return the repository path for a recognized absolute URL, or None.""" + if (not isinstance(url, str) or not url + or any(character.isspace() or ord(character) < 32 for character in url) + or "\\" in url): + return None if "://" in url: - parsed = urlsplit(url) + try: + parsed = urlsplit(url) + host = parsed.hostname + except ValueError: + return None if parsed.scheme not in {"http", "https", "ssh", "git"}: return None - host = parsed.hostname if host not in HOSTS or parsed.query or parsed.fragment: return None path = parsed.path.lstrip("/") @@ -118,6 +126,7 @@ def fix(repository, dry_run=False): def main(): parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", action="version", version=__version__) parser.add_argument("repository", nargs="?", default=".", help="repository directory (default: current directory)") parser.add_argument("--dry-run", action="store_true", diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f6c3cc5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools==80.9.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "repo-fix" +dynamic = ["version"] +description = "Normalize Dandaedre Git submodule URLs" +readme = "README.md" +requires-python = ">=3.10" +license = "MIT" +authors = [{ name = "Chado" }] +dependencies = [] + +[project.scripts] +repo-fix = "fix_submodules:main" + +[tool.setuptools] +py-modules = ["fix_submodules"] + +[tool.setuptools.dynamic] +version = { attr = "fix_submodules.__version__" } + +[tool.coverage.run] +branch = true +source = ["fix_submodules"] + +[tool.coverage.report] +fail_under = 80 +show_missing = true +skip_covered = true + +[tool.pip-licenses] +from = "expression" +allow-only = "MIT" diff --git a/requirements-ci.txt b/requirements-ci.txt new file mode 100644 index 0000000..10303b0 --- /dev/null +++ b/requirements-ci.txt @@ -0,0 +1,4 @@ +coverage==7.16.1 +pip-audit==2.10.1 +pip-licenses==5.5.5 +setuptools==80.9.0 diff --git a/tests/test_fix_submodules.py b/tests/test_fix_submodules.py index 15c0ef2..873264d 100644 --- a/tests/test_fix_submodules.py +++ b/tests/test_fix_submodules.py @@ -1,10 +1,14 @@ import contextlib import io from pathlib import Path +import random +import string +import subprocess +import sys import tempfile import unittest -from fix_submodules import fix, git, read_urls, repository_path +from fix_submodules import __version__, fix, git, read_urls, repository_path class FixTests(unittest.TestCase): @@ -60,6 +64,7 @@ class FixTests(unittest.TestCase): source = self.root / 'source' git('init', str(source)) git('-c', 'user.name=Test', '-c', 'user.email=test@example.com', + '-c', 'commit.gpgSign=false', 'commit', '--allow-empty', '-m', 'initial', cwd=source) git('-c', 'protocol.file.allow=always', 'submodule', 'add', str(source), 'checkout', cwd=self.root) @@ -109,6 +114,7 @@ class FixTests(unittest.TestCase): def test_linked_worktree(self): git('-c', 'user.name=Test', '-c', 'user.email=test@example.com', + '-c', 'commit.gpgSign=false', 'commit', '--allow-empty', '-m', 'initial', cwd=self.root) linked = self.root / 'linked' git('worktree', 'add', '-b', 'linked', str(linked), cwd=self.root) @@ -128,6 +134,38 @@ class FixTests(unittest.TestCase): 'https://dandaedre.com.evil.example/owner/repo.git'): self.assertIsNone(repository_path(url)) + def test_malformed_urls_are_rejected(self): + malformed = ( + None, 42, b'git@git.dandaedre.com:owner/repo.git', '', ' ', + 'https://[broken/owner/repo.git', + 'https://git.dandaedre.com/owner/repo.git?download=1', + 'git@git.dandaedre.com:owner\\repo.git', + 'git@git.dandaedre.com:owner/repo.git\nother', + 'git@git.dandaedre.com:owner/../repo.git', + ) + for value in malformed: + with self.subTest(value=value): + self.assertIsNone(repository_path(value)) + + def test_fuzzed_url_shapes_do_not_raise(self): + randomizer = random.Random(0) + alphabet = string.ascii_letters + string.digits + ':/.@?#[]\\\x00\n\t' + for _ in range(500): + value = ''.join(randomizer.choices(alphabet, + k=randomizer.randrange(0, 100))) + result = repository_path(value) + if result is not None: + self.assertGreaterEqual(len(result.split('/')), 2) + self.assertNotIn('..', result.split('/')) + + def test_cli_version(self): + script = Path(__file__).resolve().parents[1] / 'fix_submodules.py' + result = subprocess.run( + [sys.executable, str(script), '--version'], + text=True, capture_output=True, check=True, + ) + self.assertEqual(result.stdout.strip(), __version__) + if __name__ == '__main__': unittest.main()