Add submodule URL repair script and Jenkins test pipeline
Chado/repo-fix/pipeline/head This commit looks good Details

This commit is contained in:
Dandaedre 2026-09-25 10:43:45 +02:00
parent 3e8fb15c0e
commit 1b096b3019
5 changed files with 397 additions and 1 deletions

49
Jenkinsfile vendored Normal file
View File

@ -0,0 +1,49 @@
// Optional Gitea webhook: http://192.168.1.37:8080/gitea-webhook/post
pipeline {
agent { label 'rust_no_docker' }
options {
skipDefaultCheckout(true)
timeout(time: 10, unit: 'MINUTES')
disableConcurrentBuilds()
}
parameters {
booleanParam(name: 'RUN_CLONE_TEST', defaultValue: true,
description: 'Test URL repair on a temporary Ravenous clone (requires Gitea access).')
}
environment {
GIT_TERMINAL_PROMPT = '0'
PYTHONUNBUFFERED = '1'
}
stages {
stage('Checkout') {
steps {
checkout scm
}
}
stage('Check tools') {
steps {
sh 'python3 --version && git --version'
}
}
stage('Unit tests') {
steps {
sh 'python3 -m unittest discover -s tests -v'
}
}
stage('Ravenous clone test') {
when { expression { params.RUN_CLONE_TEST } }
steps {
sh 'python3 -m unittest discover -s integration -v'
}
}
}
post {
success { echo 'repo-fix tests passed.' }
failure { echo 'repo-fix tests failed; see the test output above.' }
}
}

View File

@ -1,3 +1,78 @@
# repo-fix
Fix repository submodules for the CI
Fix Dandaedre Gitea submodule URLs after cloning, pulling, or syncing:
- `.gitmodules` uses HTTPS for CI: `https://proxy.dandaedre.com/gitea/Beffroi/reportist.git`
- `.git/config` uses SSH locally: `git@git.dandaedre.com:Beffroi/reportist.git`
Requires Python 3 and Git on your PATH. No pip packages are needed; the script
uses the Python standard library and Git's config commands.
Preview the changes, then apply them:
```bash
python3 fix_submodules.py /path/to/repository --dry-run
python3 fix_submodules.py /path/to/repository
```
The repository defaults to the current directory, and may also be a directory
inside a working tree. Run the script again whenever a pull or `git submodule
sync` changes the URLs. Repeated runs are safe. Dry runs do not write files.
Recognizes HTTP(S), SSH, Git protocol, and SCP-style URLs on `dandaedre.com`,
`git.dandaedre.com`, and `proxy.dandaedre.com`. Old URLs such as
`git@dandaedre.com:Beffroi/reportist.git` are corrected. Repository paths and
`.git` suffixes are preserved; the proxy's `/gitea/` prefix is translated.
Missing local URL entries are added from `.gitmodules`. Existing local URLs
retain their own repository paths. Other hosts and relative URLs are skipped.
Unrelated configuration, including `active`, is preserved.
Works with `.git` directories and pointer files, including linked worktrees
(whose local config is shared). Only the selected repository's submodule
entries are changed. The script does not recurse, change `origin` remotes, or
fetch anything. Run it separately inside submodules with their own declarations.
Commit `.gitmodules` changes for CI. Local config changes are not committed.
After fixing URLs, initialize submodules with:
```bash
git -C /path/to/repository submodule update --init --recursive
```
Run tests:
```bash
python3 -m unittest discover -s tests -v
```
## Jenkins test CI
The `Jenkinsfile` uses the `rust_no_docker` agent from the Ravenous pipeline.
Configure a Jenkins Pipeline from SCM (or a multibranch pipeline) for this
repository with script path `Jenkinsfile`. The agent needs `python3` and `git`
on PATH and network access to the Gitea HTTPS proxy. No uv, Docker, or package
installation is required. Use the Gitea webhook URL from the Jenkinsfile if
your Jenkins Gitea integration is configured at that address.
Every build runs the unit tests. By default, it also shallow-clones
`Ravenous/ravenous_roll20.git` through the HTTPS proxy into a temporary directory
and verifies dry-run safety, both URL formats, repeatability, recovery after
`git submodule sync`, and correction of the old SSH hostname. The clone is
removed after the test. It does not fetch submodules or run code from Ravenous;
SSH connectivity and recursive submodule downloads are not tested.
Run the same network test locally:
```bash
python3 -m unittest discover -s integration -v
```
The HTTPS clone URL corresponds to
`git@dandaedre.com:Ravenous/ravenous_roll20.git` and avoids requiring an SSH key
for this CI test. If authentication is required, configure Git credentials on
the agent; prompts are disabled. Clone failures fail the build. Disable the
`RUN_CLONE_TEST` build parameter for an offline unit-test-only run.
This pipeline only checks out and tests. It does not publish artifacts, send
checks, push commits, or deploy files. Test results appear in the console log.

115
fix_submodules.py Normal file
View File

@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""Normalize Dandaedre submodule URLs for CI and local SSH access."""
import argparse
from pathlib import Path
import re
import subprocess
import sys
from urllib.parse import urlsplit
HTTPS_BASE = "https://proxy.dandaedre.com/gitea/"
SSH_BASE = "git@git.dandaedre.com:"
HOSTS = {"dandaedre.com", "git.dandaedre.com", "proxy.dandaedre.com"}
def repository_path(url):
"""Return the repository path for a recognized absolute URL, or None."""
if "://" in url:
parsed = urlsplit(url)
if parsed.scheme not in {"http", "https", "ssh", "git"}:
return None
host = parsed.hostname
if host not in HOSTS or parsed.query or parsed.fragment:
return None
path = parsed.path.lstrip("/")
else:
match = re.fullmatch(r"(?:[^@/:]+@)?([^/:]+):(.+)", url)
if not match or match[1].lower() not in HOSTS:
return None
host, path = match[1].lower(), match[2].lstrip("/")
if host == "proxy.dandaedre.com" and path.startswith("gitea/"):
path = path[len("gitea/"):]
if len(path.split("/")) < 2 or any(p in {"", ".", ".."} for p in path.split("/")):
return None
return path
def git(*args, cwd=None, allow_missing=False):
result = subprocess.run(
["git", *args], cwd=cwd, text=True, capture_output=True, check=False
)
if result.returncode and not (allow_missing and result.returncode == 1):
raise RuntimeError(result.stderr.strip() or "Git command failed")
return result.stdout
def read_urls(path):
if not path.exists():
return {}
output = git("config", "--file", str(path), "--null", "--get-regexp",
r"^submodule\..*\.url$", allow_missing=True)
urls = {}
for record in output.split("\0"):
if record:
key, value = record.split("\n", 1)
if key in urls:
raise RuntimeError("Duplicate submodule URL key in {}: {}".format(path, key))
urls[key] = value
return urls
def fix(repository, dry_run=False):
root = Path(git("rev-parse", "--show-toplevel", cwd=repository).strip())
common = Path(git("rev-parse", "--git-common-dir", cwd=root).strip())
if not common.is_absolute():
common = root / common
modules_file = root / ".gitmodules"
config_file = common.resolve() / "config"
modules = read_urls(modules_file)
local = read_urls(config_file)
changes = []
for key, old in modules.items():
path = repository_path(old)
if path is None:
print("Skipping unrecognized URL in .gitmodules: {}".format(key))
continue
new = HTTPS_BASE + path
if old != new:
changes.append((modules_file, key, old, new))
if key not in local:
changes.append((config_file, key, None, SSH_BASE + path))
for key, old in local.items():
path = repository_path(old)
if path is None:
print("Skipping unrecognized local URL: {}".format(key))
continue
new = SSH_BASE + path
if old != new:
changes.append((config_file, key, old, new))
for file, key, old, new in changes:
print("{}: {}\n {} -> {}".format(file, key, old or "(missing)", new))
if not dry_run:
git("config", "--file", str(file), "--replace-all", key, new)
print("{} {} URL(s).".format("Would update" if dry_run else "Updated", len(changes)))
return len(changes)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("repository", nargs="?", default=".",
help="repository directory (default: current directory)")
parser.add_argument("--dry-run", action="store_true",
help="show planned changes without writing files")
args = parser.parse_args()
try:
fix(args.repository, args.dry_run)
except (OSError, RuntimeError, ValueError) as error:
print("Error: {}".format(error), file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

75
integration/test_clone.py Normal file
View File

@ -0,0 +1,75 @@
"""Opt-in network test; run with unittest discover -s integration -v."""
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
from fix_submodules import git, read_urls, repository_path
SCRIPT = Path(__file__).resolve().parents[1] / 'fix_submodules.py'
CLONE_URL = 'https://proxy.dandaedre.com/gitea/Ravenous/ravenous_roll20.git'
class CloneTest(unittest.TestCase):
def test_real_repository(self):
with tempfile.TemporaryDirectory(prefix='repo-fix-ci-') as directory:
root = Path(directory) / 'ravenous'
subprocess.run(
['git', 'clone', '--depth', '1', '--no-recurse-submodules', CLONE_URL, str(root)],
env=dict(os.environ, GIT_TERMINAL_PROMPT='0'), check=True, timeout=120,
)
modules = root / '.gitmodules'
config = root / '.git/config'
original = read_urls(modules)
paths = {key: repository_path(url) for key, url in original.items()
if repository_path(url) is not None}
self.assertTrue(paths, 'Expected at least one Dandaedre submodule to exercise')
def run_fix(*args):
subprocess.run([sys.executable, str(SCRIPT), str(root), *args],
check=True, timeout=30)
def snapshot():
return modules.read_bytes(), config.read_bytes()
def check_urls():
public, local = read_urls(modules), read_urls(config)
for key, path in paths.items():
self.assertEqual(public[key], 'https://proxy.dandaedre.com/gitea/' + path)
self.assertEqual(local[key], 'git@git.dandaedre.com:' + path)
for key, url in original.items():
if key not in paths:
self.assertEqual(public[key], url)
before = snapshot()
run_fix('--dry-run')
self.assertEqual(snapshot(), before, 'Dry run changed files')
run_fix()
check_urls()
fixed = snapshot()
run_fix()
self.assertEqual(snapshot(), fixed, 'Second run changed files')
# Reproduce Git overwriting local URLs with the CI HTTPS URLs.
git('submodule', 'init', cwd=root)
git('submodule', 'sync', cwd=root)
for key, path in paths.items():
self.assertEqual(read_urls(config)[key],
'https://proxy.dandaedre.com/gitea/' + path)
run_fix()
check_urls()
# Exercise legacy hostnames using real submodule names and paths.
for file in (modules, config):
for key, path in paths.items():
git('config', '--file', str(file), key, 'git@dandaedre.com:' + path)
run_fix()
check_urls()
if __name__ == '__main__':
unittest.main()

View File

@ -0,0 +1,82 @@
import contextlib
import io
from pathlib import Path
import tempfile
import unittest
from fix_submodules import fix, git, read_urls, repository_path
class FixTests(unittest.TestCase):
def setUp(self):
directory = tempfile.TemporaryDirectory()
self.addCleanup(directory.cleanup)
self.root = Path(directory.name)
git('init', str(self.root))
self.modules = self.root / '.gitmodules'
self.config = self.root / '.git/config'
self.modules.write_text(
'# Keep this comment\n[submodule "reportist"]\n'
'\tpath = reportist\n'
'\turl = git@dandaedre.com:Beffroi/reportist.git\n'
'[submodule "other"]\n\tpath = other\n'
'\turl = https://example.com/owner/other.git\n'
)
git('config', '--file', str(self.config), 'submodule.reportist.active', 'true')
def run_fix(self, **kwargs):
with contextlib.redirect_stdout(io.StringIO()):
return fix(self.root, **kwargs)
def test_dry_run_apply_and_repeat(self):
before = (self.modules.read_bytes(), self.config.read_bytes())
self.assertEqual(self.run_fix(dry_run=True), 2)
self.assertEqual(before, (self.modules.read_bytes(), self.config.read_bytes()))
self.assertEqual(self.run_fix(), 2)
self.assertEqual(read_urls(self.modules)['submodule.reportist.url'],
'https://proxy.dandaedre.com/gitea/Beffroi/reportist.git')
self.assertEqual(read_urls(self.config)['submodule.reportist.url'],
'git@git.dandaedre.com:Beffroi/reportist.git')
self.assertIn('# Keep this comment', self.modules.read_text())
self.assertIn('active = true', self.config.read_text())
self.assertEqual(read_urls(self.modules)['submodule.other.url'],
'https://example.com/owner/other.git')
self.assertEqual(self.run_fix(), 0)
def test_existing_local_path(self):
for url in ('https://proxy.dandaedre.com/gitea/Beffroi/fork.git',
'git@dandaedre.com:Beffroi/fork.git'):
with self.subTest(url=url):
git('config', '--file', str(self.config), 'submodule.reportist.url', url)
self.run_fix()
self.assertEqual(read_urls(self.config)['submodule.reportist.url'],
'git@git.dandaedre.com:Beffroi/fork.git')
def test_missing_gitmodules(self):
self.modules.unlink()
self.assertEqual(self.run_fix(), 0)
def test_linked_worktree(self):
git('-c', 'user.name=Test', '-c', 'user.email=test@example.com',
'commit', '--allow-empty', '-m', 'initial', cwd=self.root)
linked = self.root / 'linked'
git('worktree', 'add', '-b', 'linked', str(linked), cwd=self.root)
(linked / '.gitmodules').write_bytes(self.modules.read_bytes())
with contextlib.redirect_stdout(io.StringIO()):
self.assertEqual(fix(linked), 2)
self.assertEqual(read_urls(self.config)['submodule.reportist.url'],
'git@git.dandaedre.com:Beffroi/reportist.git')
def test_url_forms(self):
for url in ('git@git.dandaedre.com:Beffroi/reportist.git',
'ssh://git@dandaedre.com/Beffroi/reportist.git',
'https://proxy.dandaedre.com/gitea/Beffroi/reportist.git',
'git://git.dandaedre.com/Beffroi/reportist.git'):
self.assertEqual(repository_path(url), 'Beffroi/reportist.git')
for url in ('../other.git', 'git@example.com:owner/repo.git',
'https://dandaedre.com.evil.example/owner/repo.git'):
self.assertIsNone(repository_path(url))
if __name__ == '__main__':
unittest.main()