Add Python CI quality and dependency gates
Chado/repo-fix/pipeline/head This commit looks good
Details
Chado/repo-fix/pipeline/head This commit looks good
Details
This commit is contained in:
parent
b42d9f5f59
commit
a80eeafb9e
|
|
@ -123,6 +123,7 @@ celerybeat.pid
|
|||
# Environments
|
||||
.env
|
||||
.venv
|
||||
.venv-app/
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
|
|
@ -159,4 +160,3 @@ cython_debug/
|
|||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ pipeline {
|
|||
environment {
|
||||
GIT_TERMINAL_PROMPT = '0'
|
||||
PYTHONUNBUFFERED = '1'
|
||||
PIP_DISABLE_PIP_VERSION_CHECK = '1'
|
||||
}
|
||||
|
||||
stages {
|
||||
|
|
@ -29,9 +30,35 @@ pipeline {
|
|||
sh 'python3 --version && git --version'
|
||||
}
|
||||
}
|
||||
stage('Unit tests') {
|
||||
stage('Set up Python tools') {
|
||||
steps {
|
||||
sh 'python3 -m unittest discover -s tests -v'
|
||||
sh '''
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-ci.txt
|
||||
python3 -m venv .venv-app
|
||||
.venv-app/bin/python -m pip install setuptools==80.9.0
|
||||
.venv-app/bin/python -m pip install --no-build-isolation --no-deps .
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Dependency vulnerability audit') {
|
||||
steps {
|
||||
sh '.venv/bin/pip-audit --strict .'
|
||||
}
|
||||
}
|
||||
stage('Dependency license policy') {
|
||||
steps {
|
||||
sh '.venv/bin/pip-licenses --python=.venv-app/bin/python'
|
||||
}
|
||||
}
|
||||
stage('Unit tests and coverage') {
|
||||
steps {
|
||||
sh '''
|
||||
.venv/bin/coverage erase
|
||||
.venv/bin/coverage run -m unittest discover -s tests -v
|
||||
.venv/bin/coverage report
|
||||
.venv/bin/coverage xml
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Ravenous clone test') {
|
||||
|
|
@ -43,7 +70,11 @@ pipeline {
|
|||
}
|
||||
|
||||
post {
|
||||
success { echo 'repo-fix tests passed.' }
|
||||
failure { echo 'repo-fix tests failed; see the test output above.' }
|
||||
success {
|
||||
echo 'repo-fix audit, license policy, coverage, and tests passed.'
|
||||
}
|
||||
failure {
|
||||
echo 'repo-fix CI failed; check the audit, policy, coverage, and test stages above.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
51
README.md
51
README.md
|
|
@ -7,7 +7,7 @@ Fix Dandaedre Gitea submodule URLs after cloning, pulling, or syncing:
|
|||
- `.git/modules/**/config` uses the same SSH format for submodule remotes
|
||||
(`url` and `pushurl`) and nested submodule URL entries.
|
||||
|
||||
Requires Python 3 and Git on your PATH. No pip packages are needed; the script
|
||||
Requires Python 3.10 or newer and Git on your PATH. No runtime pip packages are needed; the script
|
||||
uses the Python standard library and Git's config commands.
|
||||
|
||||
Preview the changes, then apply them:
|
||||
|
|
@ -17,6 +17,12 @@ python3 fix_submodules.py /path/to/repository --dry-run
|
|||
python3 fix_submodules.py /path/to/repository
|
||||
```
|
||||
|
||||
The project follows semantic versioning and currently reports version `0.1.0`:
|
||||
|
||||
```bash
|
||||
python3 fix_submodules.py --version
|
||||
```
|
||||
|
||||
The repository defaults to the current directory, and may also be a directory
|
||||
inside a working tree. Run the script again whenever a pull or `git submodule
|
||||
sync` changes the URLs. Repeated runs are safe. Dry runs do not write files.
|
||||
|
|
@ -51,16 +57,51 @@ Run tests:
|
|||
python3 -m unittest discover -s tests -v
|
||||
```
|
||||
|
||||
The unit suite includes a small deterministic fuzz test and malformed-value
|
||||
cases for URL parsing. They are intended to catch crashes and unsafe path
|
||||
shapes, not to replace a dedicated fuzzer.
|
||||
|
||||
For the same coverage check as CI, install the pinned tools and run:
|
||||
|
||||
```bash
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-ci.txt
|
||||
.venv/bin/coverage run -m unittest discover -s tests -v
|
||||
.venv/bin/coverage report
|
||||
```
|
||||
|
||||
Branch coverage must remain at or above 80%. This deliberately modest floor
|
||||
keeps the check useful without forcing low-value tests.
|
||||
|
||||
## Jenkins test CI
|
||||
|
||||
The `Jenkinsfile` uses the `rust_no_docker` agent from the Ravenous pipeline.
|
||||
Configure a Jenkins Pipeline from SCM (or a multibranch pipeline) for this
|
||||
repository with script path `Jenkinsfile`. The agent needs `python3` and `git`
|
||||
on PATH and network access to the Gitea HTTPS proxy. No uv, Docker, or package
|
||||
installation is required. Use the Gitea webhook URL from the Jenkinsfile if
|
||||
your Jenkins Gitea integration is configured at that address.
|
||||
on PATH and network access to PyPI, vulnerability data, and the Gitea HTTPS
|
||||
proxy. No uv or Docker is required; CI creates isolated virtual environments
|
||||
and installs the pinned tools. Use the Gitea webhook URL from the Jenkinsfile
|
||||
if your Jenkins Gitea integration is configured at that address.
|
||||
|
||||
Every build runs the unit tests. By default, it also shallow-clones
|
||||
Every build runs the unit tests with coverage. Before testing, it runs two
|
||||
small supply-chain gates: `pip-audit` 2.10.1 fails on known vulnerabilities or
|
||||
dependency-collection errors, and `pip-licenses` 5.5.5 allows only MIT-licensed
|
||||
installed application packages. The CI environment also pins coverage 7.16.1
|
||||
and setuptools 80.9.0 in `requirements-ci.txt`. Run the gates locally with:
|
||||
|
||||
```bash
|
||||
.venv/bin/pip-audit --strict .
|
||||
.venv/bin/pip-licenses --python=.venv-app/bin/python
|
||||
```
|
||||
|
||||
The audit needs network access for package resolution and current advisory
|
||||
data. The license gate inspects the isolated application environment; update
|
||||
the explicit allow-list only after reviewing a new dependency. The project has
|
||||
no runtime dependencies today. These checks do not analyze the standard
|
||||
library, Git, malicious packages, provenance, or undisclosed vulnerabilities;
|
||||
no broader hardening tools are in scope.
|
||||
|
||||
By default, CI also shallow-clones
|
||||
`Ravenous/ravenous_roll20.git` through the HTTPS proxy into a temporary directory
|
||||
and verifies dry-run safety, both URL formats, repeatability, recovery after
|
||||
`git submodule sync`, and correction of the old SSH hostname. The clone is
|
||||
|
|
|
|||
|
|
@ -12,15 +12,23 @@ from urllib.parse import urlsplit
|
|||
HTTPS_BASE = "https://proxy.dandaedre.com/gitea/"
|
||||
SSH_BASE = "git@git.dandaedre.com:"
|
||||
HOSTS = {"dandaedre.com", "git.dandaedre.com", "proxy.dandaedre.com"}
|
||||
__version__ = "0.1.0"
|
||||
|
||||
|
||||
def repository_path(url):
|
||||
"""Return the repository path for a recognized absolute URL, or None."""
|
||||
if (not isinstance(url, str) or not url
|
||||
or any(character.isspace() or ord(character) < 32 for character in url)
|
||||
or "\\" in url):
|
||||
return None
|
||||
if "://" in url:
|
||||
parsed = urlsplit(url)
|
||||
try:
|
||||
parsed = urlsplit(url)
|
||||
host = parsed.hostname
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.scheme not in {"http", "https", "ssh", "git"}:
|
||||
return None
|
||||
host = parsed.hostname
|
||||
if host not in HOSTS or parsed.query or parsed.fragment:
|
||||
return None
|
||||
path = parsed.path.lstrip("/")
|
||||
|
|
@ -118,6 +126,7 @@ def fix(repository, dry_run=False):
|
|||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--version", action="version", version=__version__)
|
||||
parser.add_argument("repository", nargs="?", default=".",
|
||||
help="repository directory (default: current directory)")
|
||||
parser.add_argument("--dry-run", action="store_true",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
[build-system]
|
||||
requires = ["setuptools==80.9.0"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "repo-fix"
|
||||
dynamic = ["version"]
|
||||
description = "Normalize Dandaedre Git submodule URLs"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
license = "MIT"
|
||||
authors = [{ name = "Chado" }]
|
||||
dependencies = []
|
||||
|
||||
[project.scripts]
|
||||
repo-fix = "fix_submodules:main"
|
||||
|
||||
[tool.setuptools]
|
||||
py-modules = ["fix_submodules"]
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
version = { attr = "fix_submodules.__version__" }
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
source = ["fix_submodules"]
|
||||
|
||||
[tool.coverage.report]
|
||||
fail_under = 80
|
||||
show_missing = true
|
||||
skip_covered = true
|
||||
|
||||
[tool.pip-licenses]
|
||||
from = "expression"
|
||||
allow-only = "MIT"
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
coverage==7.16.1
|
||||
pip-audit==2.10.1
|
||||
pip-licenses==5.5.5
|
||||
setuptools==80.9.0
|
||||
|
|
@ -1,10 +1,14 @@
|
|||
import contextlib
|
||||
import io
|
||||
from pathlib import Path
|
||||
import random
|
||||
import string
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from fix_submodules import fix, git, read_urls, repository_path
|
||||
from fix_submodules import __version__, fix, git, read_urls, repository_path
|
||||
|
||||
|
||||
class FixTests(unittest.TestCase):
|
||||
|
|
@ -60,6 +64,7 @@ class FixTests(unittest.TestCase):
|
|||
source = self.root / 'source'
|
||||
git('init', str(source))
|
||||
git('-c', 'user.name=Test', '-c', 'user.email=test@example.com',
|
||||
'-c', 'commit.gpgSign=false',
|
||||
'commit', '--allow-empty', '-m', 'initial', cwd=source)
|
||||
git('-c', 'protocol.file.allow=always', 'submodule', 'add',
|
||||
str(source), 'checkout', cwd=self.root)
|
||||
|
|
@ -109,6 +114,7 @@ class FixTests(unittest.TestCase):
|
|||
|
||||
def test_linked_worktree(self):
|
||||
git('-c', 'user.name=Test', '-c', 'user.email=test@example.com',
|
||||
'-c', 'commit.gpgSign=false',
|
||||
'commit', '--allow-empty', '-m', 'initial', cwd=self.root)
|
||||
linked = self.root / 'linked'
|
||||
git('worktree', 'add', '-b', 'linked', str(linked), cwd=self.root)
|
||||
|
|
@ -128,6 +134,38 @@ class FixTests(unittest.TestCase):
|
|||
'https://dandaedre.com.evil.example/owner/repo.git'):
|
||||
self.assertIsNone(repository_path(url))
|
||||
|
||||
def test_malformed_urls_are_rejected(self):
|
||||
malformed = (
|
||||
None, 42, b'git@git.dandaedre.com:owner/repo.git', '', ' ',
|
||||
'https://[broken/owner/repo.git',
|
||||
'https://git.dandaedre.com/owner/repo.git?download=1',
|
||||
'git@git.dandaedre.com:owner\\repo.git',
|
||||
'git@git.dandaedre.com:owner/repo.git\nother',
|
||||
'git@git.dandaedre.com:owner/../repo.git',
|
||||
)
|
||||
for value in malformed:
|
||||
with self.subTest(value=value):
|
||||
self.assertIsNone(repository_path(value))
|
||||
|
||||
def test_fuzzed_url_shapes_do_not_raise(self):
|
||||
randomizer = random.Random(0)
|
||||
alphabet = string.ascii_letters + string.digits + ':/.@?#[]\\\x00\n\t'
|
||||
for _ in range(500):
|
||||
value = ''.join(randomizer.choices(alphabet,
|
||||
k=randomizer.randrange(0, 100)))
|
||||
result = repository_path(value)
|
||||
if result is not None:
|
||||
self.assertGreaterEqual(len(result.split('/')), 2)
|
||||
self.assertNotIn('..', result.split('/'))
|
||||
|
||||
def test_cli_version(self):
|
||||
script = Path(__file__).resolve().parents[1] / 'fix_submodules.py'
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(script), '--version'],
|
||||
text=True, capture_output=True, check=True,
|
||||
)
|
||||
self.assertEqual(result.stdout.strip(), __version__)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue