Add Python CI quality and dependency gates
Chado/repo-fix/pipeline/head This commit looks good Details

This commit is contained in:
Dandaedre 2026-10-03 22:37:33 +02:00
parent b42d9f5f59
commit a80eeafb9e
7 changed files with 171 additions and 13 deletions

2
.gitignore vendored
View File

@ -123,6 +123,7 @@ celerybeat.pid
# Environments
.env
.venv
.venv-app/
env/
venv/
ENV/
@ -159,4 +160,3 @@ cython_debug/
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/

39
Jenkinsfile vendored
View File

@ -16,6 +16,7 @@ pipeline {
environment {
GIT_TERMINAL_PROMPT = '0'
PYTHONUNBUFFERED = '1'
PIP_DISABLE_PIP_VERSION_CHECK = '1'
}
stages {
@ -29,9 +30,35 @@ pipeline {
sh 'python3 --version && git --version'
}
}
stage('Unit tests') {
stage('Set up Python tools') {
steps {
sh 'python3 -m unittest discover -s tests -v'
sh '''
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-ci.txt
python3 -m venv .venv-app
.venv-app/bin/python -m pip install setuptools==80.9.0
.venv-app/bin/python -m pip install --no-build-isolation --no-deps .
'''
}
}
stage('Dependency vulnerability audit') {
steps {
sh '.venv/bin/pip-audit --strict .'
}
}
stage('Dependency license policy') {
steps {
sh '.venv/bin/pip-licenses --python=.venv-app/bin/python'
}
}
stage('Unit tests and coverage') {
steps {
sh '''
.venv/bin/coverage erase
.venv/bin/coverage run -m unittest discover -s tests -v
.venv/bin/coverage report
.venv/bin/coverage xml
'''
}
}
stage('Ravenous clone test') {
@ -43,7 +70,11 @@ pipeline {
}
post {
success { echo 'repo-fix tests passed.' }
failure { echo 'repo-fix tests failed; see the test output above.' }
success {
echo 'repo-fix audit, license policy, coverage, and tests passed.'
}
failure {
echo 'repo-fix CI failed; check the audit, policy, coverage, and test stages above.'
}
}
}

View File

@ -7,7 +7,7 @@ Fix Dandaedre Gitea submodule URLs after cloning, pulling, or syncing:
- `.git/modules/**/config` uses the same SSH format for submodule remotes
(`url` and `pushurl`) and nested submodule URL entries.
Requires Python 3 and Git on your PATH. No pip packages are needed; the script
Requires Python 3.10 or newer and Git on your PATH. No runtime pip packages are needed; the script
uses the Python standard library and Git's config commands.
Preview the changes, then apply them:
@ -17,6 +17,12 @@ python3 fix_submodules.py /path/to/repository --dry-run
python3 fix_submodules.py /path/to/repository
```
The project follows semantic versioning and currently reports version `0.1.0`:
```bash
python3 fix_submodules.py --version
```
The repository defaults to the current directory, and may also be a directory
inside a working tree. Run the script again whenever a pull or `git submodule
sync` changes the URLs. Repeated runs are safe. Dry runs do not write files.
@ -51,16 +57,51 @@ Run tests:
python3 -m unittest discover -s tests -v
```
The unit suite includes a small deterministic fuzz test and malformed-value
cases for URL parsing. They are intended to catch crashes and unsafe path
shapes, not to replace a dedicated fuzzer.
For the same coverage check as CI, install the pinned tools and run:
```bash
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-ci.txt
.venv/bin/coverage run -m unittest discover -s tests -v
.venv/bin/coverage report
```
Branch coverage must remain at or above 80%. This deliberately modest floor
keeps the check useful without forcing low-value tests.
## Jenkins test CI
The `Jenkinsfile` uses the `rust_no_docker` agent from the Ravenous pipeline.
Configure a Jenkins Pipeline from SCM (or a multibranch pipeline) for this
repository with script path `Jenkinsfile`. The agent needs `python3` and `git`
on PATH and network access to the Gitea HTTPS proxy. No uv, Docker, or package
installation is required. Use the Gitea webhook URL from the Jenkinsfile if
your Jenkins Gitea integration is configured at that address.
on PATH and network access to PyPI, vulnerability data, and the Gitea HTTPS
proxy. No uv or Docker is required; CI creates isolated virtual environments
and installs the pinned tools. Use the Gitea webhook URL from the Jenkinsfile
if your Jenkins Gitea integration is configured at that address.
Every build runs the unit tests. By default, it also shallow-clones
Every build runs the unit tests with coverage. Before testing, it runs two
small supply-chain gates: `pip-audit` 2.10.1 fails on known vulnerabilities or
dependency-collection errors, and `pip-licenses` 5.5.5 allows only MIT-licensed
installed application packages. The CI environment also pins coverage 7.16.1
and setuptools 80.9.0 in `requirements-ci.txt`. Run the gates locally with:
```bash
.venv/bin/pip-audit --strict .
.venv/bin/pip-licenses --python=.venv-app/bin/python
```
The audit needs network access for package resolution and current advisory
data. The license gate inspects the isolated application environment; update
the explicit allow-list only after reviewing a new dependency. The project has
no runtime dependencies today. These checks do not analyze the standard
library, Git, malicious packages, provenance, or undisclosed vulnerabilities;
no broader hardening tools are in scope.
By default, CI also shallow-clones
`Ravenous/ravenous_roll20.git` through the HTTPS proxy into a temporary directory
and verifies dry-run safety, both URL formats, repeatability, recovery after
`git submodule sync`, and correction of the old SSH hostname. The clone is

View File

@ -12,15 +12,23 @@ from urllib.parse import urlsplit
HTTPS_BASE = "https://proxy.dandaedre.com/gitea/"
SSH_BASE = "git@git.dandaedre.com:"
HOSTS = {"dandaedre.com", "git.dandaedre.com", "proxy.dandaedre.com"}
__version__ = "0.1.0"
def repository_path(url):
"""Return the repository path for a recognized absolute URL, or None."""
if (not isinstance(url, str) or not url
or any(character.isspace() or ord(character) < 32 for character in url)
or "\\" in url):
return None
if "://" in url:
parsed = urlsplit(url)
try:
parsed = urlsplit(url)
host = parsed.hostname
except ValueError:
return None
if parsed.scheme not in {"http", "https", "ssh", "git"}:
return None
host = parsed.hostname
if host not in HOSTS or parsed.query or parsed.fragment:
return None
path = parsed.path.lstrip("/")
@ -118,6 +126,7 @@ def fix(repository, dry_run=False):
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--version", action="version", version=__version__)
parser.add_argument("repository", nargs="?", default=".",
help="repository directory (default: current directory)")
parser.add_argument("--dry-run", action="store_true",

35
pyproject.toml Normal file
View File

@ -0,0 +1,35 @@
[build-system]
requires = ["setuptools==80.9.0"]
build-backend = "setuptools.build_meta"
[project]
name = "repo-fix"
dynamic = ["version"]
description = "Normalize Dandaedre Git submodule URLs"
readme = "README.md"
requires-python = ">=3.10"
license = "MIT"
authors = [{ name = "Chado" }]
dependencies = []
[project.scripts]
repo-fix = "fix_submodules:main"
[tool.setuptools]
py-modules = ["fix_submodules"]
[tool.setuptools.dynamic]
version = { attr = "fix_submodules.__version__" }
[tool.coverage.run]
branch = true
source = ["fix_submodules"]
[tool.coverage.report]
fail_under = 80
show_missing = true
skip_covered = true
[tool.pip-licenses]
from = "expression"
allow-only = "MIT"

4
requirements-ci.txt Normal file
View File

@ -0,0 +1,4 @@
coverage==7.16.1
pip-audit==2.10.1
pip-licenses==5.5.5
setuptools==80.9.0

View File

@ -1,10 +1,14 @@
import contextlib
import io
from pathlib import Path
import random
import string
import subprocess
import sys
import tempfile
import unittest
from fix_submodules import fix, git, read_urls, repository_path
from fix_submodules import __version__, fix, git, read_urls, repository_path
class FixTests(unittest.TestCase):
@ -60,6 +64,7 @@ class FixTests(unittest.TestCase):
source = self.root / 'source'
git('init', str(source))
git('-c', 'user.name=Test', '-c', 'user.email=test@example.com',
'-c', 'commit.gpgSign=false',
'commit', '--allow-empty', '-m', 'initial', cwd=source)
git('-c', 'protocol.file.allow=always', 'submodule', 'add',
str(source), 'checkout', cwd=self.root)
@ -109,6 +114,7 @@ class FixTests(unittest.TestCase):
def test_linked_worktree(self):
git('-c', 'user.name=Test', '-c', 'user.email=test@example.com',
'-c', 'commit.gpgSign=false',
'commit', '--allow-empty', '-m', 'initial', cwd=self.root)
linked = self.root / 'linked'
git('worktree', 'add', '-b', 'linked', str(linked), cwd=self.root)
@ -128,6 +134,38 @@ class FixTests(unittest.TestCase):
'https://dandaedre.com.evil.example/owner/repo.git'):
self.assertIsNone(repository_path(url))
def test_malformed_urls_are_rejected(self):
malformed = (
None, 42, b'git@git.dandaedre.com:owner/repo.git', '', ' ',
'https://[broken/owner/repo.git',
'https://git.dandaedre.com/owner/repo.git?download=1',
'git@git.dandaedre.com:owner\\repo.git',
'git@git.dandaedre.com:owner/repo.git\nother',
'git@git.dandaedre.com:owner/../repo.git',
)
for value in malformed:
with self.subTest(value=value):
self.assertIsNone(repository_path(value))
def test_fuzzed_url_shapes_do_not_raise(self):
randomizer = random.Random(0)
alphabet = string.ascii_letters + string.digits + ':/.@?#[]\\\x00\n\t'
for _ in range(500):
value = ''.join(randomizer.choices(alphabet,
k=randomizer.randrange(0, 100)))
result = repository_path(value)
if result is not None:
self.assertGreaterEqual(len(result.split('/')), 2)
self.assertNotIn('..', result.split('/'))
def test_cli_version(self):
script = Path(__file__).resolve().parents[1] / 'fix_submodules.py'
result = subprocess.run(
[sys.executable, str(script), '--version'],
text=True, capture_output=True, check=True,
)
self.assertEqual(result.stdout.strip(), __version__)
if __name__ == '__main__':
unittest.main()