Add submodule URL repair script and Jenkins test pipeline
Chado/repo-fix/pipeline/head This commit looks good
Details
Chado/repo-fix/pipeline/head This commit looks good
Details
This commit is contained in:
parent
3e8fb15c0e
commit
1b096b3019
|
|
@ -0,0 +1,49 @@
|
|||
// Optional Gitea webhook: http://192.168.1.37:8080/gitea-webhook/post
|
||||
pipeline {
|
||||
agent { label 'rust_no_docker' }
|
||||
|
||||
options {
|
||||
skipDefaultCheckout(true)
|
||||
timeout(time: 10, unit: 'MINUTES')
|
||||
disableConcurrentBuilds()
|
||||
}
|
||||
|
||||
parameters {
|
||||
booleanParam(name: 'RUN_CLONE_TEST', defaultValue: true,
|
||||
description: 'Test URL repair on a temporary Ravenous clone (requires Gitea access).')
|
||||
}
|
||||
|
||||
environment {
|
||||
GIT_TERMINAL_PROMPT = '0'
|
||||
PYTHONUNBUFFERED = '1'
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Checkout') {
|
||||
steps {
|
||||
checkout scm
|
||||
}
|
||||
}
|
||||
stage('Check tools') {
|
||||
steps {
|
||||
sh 'python3 --version && git --version'
|
||||
}
|
||||
}
|
||||
stage('Unit tests') {
|
||||
steps {
|
||||
sh 'python3 -m unittest discover -s tests -v'
|
||||
}
|
||||
}
|
||||
stage('Ravenous clone test') {
|
||||
when { expression { params.RUN_CLONE_TEST } }
|
||||
steps {
|
||||
sh 'python3 -m unittest discover -s integration -v'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
success { echo 'repo-fix tests passed.' }
|
||||
failure { echo 'repo-fix tests failed; see the test output above.' }
|
||||
}
|
||||
}
|
||||
77
README.md
77
README.md
|
|
@ -1,3 +1,78 @@
|
|||
# repo-fix
|
||||
|
||||
Fix repository submodules for the CI
|
||||
Fix Dandaedre Gitea submodule URLs after cloning, pulling, or syncing:
|
||||
|
||||
- `.gitmodules` uses HTTPS for CI: `https://proxy.dandaedre.com/gitea/Beffroi/reportist.git`
|
||||
- `.git/config` uses SSH locally: `git@git.dandaedre.com:Beffroi/reportist.git`
|
||||
|
||||
Requires Python 3 and Git on your PATH. No pip packages are needed; the script
|
||||
uses the Python standard library and Git's config commands.
|
||||
|
||||
Preview the changes, then apply them:
|
||||
|
||||
```bash
|
||||
python3 fix_submodules.py /path/to/repository --dry-run
|
||||
python3 fix_submodules.py /path/to/repository
|
||||
```
|
||||
|
||||
The repository defaults to the current directory, and may also be a directory
|
||||
inside a working tree. Run the script again whenever a pull or `git submodule
|
||||
sync` changes the URLs. Repeated runs are safe. Dry runs do not write files.
|
||||
|
||||
Recognizes HTTP(S), SSH, Git protocol, and SCP-style URLs on `dandaedre.com`,
|
||||
`git.dandaedre.com`, and `proxy.dandaedre.com`. Old URLs such as
|
||||
`git@dandaedre.com:Beffroi/reportist.git` are corrected. Repository paths and
|
||||
`.git` suffixes are preserved; the proxy's `/gitea/` prefix is translated.
|
||||
|
||||
Missing local URL entries are added from `.gitmodules`. Existing local URLs
|
||||
retain their own repository paths. Other hosts and relative URLs are skipped.
|
||||
Unrelated configuration, including `active`, is preserved.
|
||||
|
||||
Works with `.git` directories and pointer files, including linked worktrees
|
||||
(whose local config is shared). Only the selected repository's submodule
|
||||
entries are changed. The script does not recurse, change `origin` remotes, or
|
||||
fetch anything. Run it separately inside submodules with their own declarations.
|
||||
|
||||
Commit `.gitmodules` changes for CI. Local config changes are not committed.
|
||||
After fixing URLs, initialize submodules with:
|
||||
|
||||
```bash
|
||||
git -C /path/to/repository submodule update --init --recursive
|
||||
```
|
||||
|
||||
Run tests:
|
||||
|
||||
```bash
|
||||
python3 -m unittest discover -s tests -v
|
||||
```
|
||||
|
||||
## Jenkins test CI
|
||||
|
||||
The `Jenkinsfile` uses the `rust_no_docker` agent from the Ravenous pipeline.
|
||||
Configure a Jenkins Pipeline from SCM (or a multibranch pipeline) for this
|
||||
repository with script path `Jenkinsfile`. The agent needs `python3` and `git`
|
||||
on PATH and network access to the Gitea HTTPS proxy. No uv, Docker, or package
|
||||
installation is required. Use the Gitea webhook URL from the Jenkinsfile if
|
||||
your Jenkins Gitea integration is configured at that address.
|
||||
|
||||
Every build runs the unit tests. By default, it also shallow-clones
|
||||
`Ravenous/ravenous_roll20.git` through the HTTPS proxy into a temporary directory
|
||||
and verifies dry-run safety, both URL formats, repeatability, recovery after
|
||||
`git submodule sync`, and correction of the old SSH hostname. The clone is
|
||||
removed after the test. It does not fetch submodules or run code from Ravenous;
|
||||
SSH connectivity and recursive submodule downloads are not tested.
|
||||
|
||||
Run the same network test locally:
|
||||
|
||||
```bash
|
||||
python3 -m unittest discover -s integration -v
|
||||
```
|
||||
|
||||
The HTTPS clone URL corresponds to
|
||||
`git@dandaedre.com:Ravenous/ravenous_roll20.git` and avoids requiring an SSH key
|
||||
for this CI test. If authentication is required, configure Git credentials on
|
||||
the agent; prompts are disabled. Clone failures fail the build. Disable the
|
||||
`RUN_CLONE_TEST` build parameter for an offline unit-test-only run.
|
||||
|
||||
This pipeline only checks out and tests. It does not publish artifacts, send
|
||||
checks, push commits, or deploy files. Test results appear in the console log.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,115 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Normalize Dandaedre submodule URLs for CI and local SSH access."""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
HTTPS_BASE = "https://proxy.dandaedre.com/gitea/"
|
||||
SSH_BASE = "git@git.dandaedre.com:"
|
||||
HOSTS = {"dandaedre.com", "git.dandaedre.com", "proxy.dandaedre.com"}
|
||||
|
||||
|
||||
def repository_path(url):
|
||||
"""Return the repository path for a recognized absolute URL, or None."""
|
||||
if "://" in url:
|
||||
parsed = urlsplit(url)
|
||||
if parsed.scheme not in {"http", "https", "ssh", "git"}:
|
||||
return None
|
||||
host = parsed.hostname
|
||||
if host not in HOSTS or parsed.query or parsed.fragment:
|
||||
return None
|
||||
path = parsed.path.lstrip("/")
|
||||
else:
|
||||
match = re.fullmatch(r"(?:[^@/:]+@)?([^/:]+):(.+)", url)
|
||||
if not match or match[1].lower() not in HOSTS:
|
||||
return None
|
||||
host, path = match[1].lower(), match[2].lstrip("/")
|
||||
if host == "proxy.dandaedre.com" and path.startswith("gitea/"):
|
||||
path = path[len("gitea/"):]
|
||||
if len(path.split("/")) < 2 or any(p in {"", ".", ".."} for p in path.split("/")):
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def git(*args, cwd=None, allow_missing=False):
|
||||
result = subprocess.run(
|
||||
["git", *args], cwd=cwd, text=True, capture_output=True, check=False
|
||||
)
|
||||
if result.returncode and not (allow_missing and result.returncode == 1):
|
||||
raise RuntimeError(result.stderr.strip() or "Git command failed")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def read_urls(path):
|
||||
if not path.exists():
|
||||
return {}
|
||||
output = git("config", "--file", str(path), "--null", "--get-regexp",
|
||||
r"^submodule\..*\.url$", allow_missing=True)
|
||||
urls = {}
|
||||
for record in output.split("\0"):
|
||||
if record:
|
||||
key, value = record.split("\n", 1)
|
||||
if key in urls:
|
||||
raise RuntimeError("Duplicate submodule URL key in {}: {}".format(path, key))
|
||||
urls[key] = value
|
||||
return urls
|
||||
|
||||
|
||||
def fix(repository, dry_run=False):
|
||||
root = Path(git("rev-parse", "--show-toplevel", cwd=repository).strip())
|
||||
common = Path(git("rev-parse", "--git-common-dir", cwd=root).strip())
|
||||
if not common.is_absolute():
|
||||
common = root / common
|
||||
modules_file = root / ".gitmodules"
|
||||
config_file = common.resolve() / "config"
|
||||
modules = read_urls(modules_file)
|
||||
local = read_urls(config_file)
|
||||
changes = []
|
||||
for key, old in modules.items():
|
||||
path = repository_path(old)
|
||||
if path is None:
|
||||
print("Skipping unrecognized URL in .gitmodules: {}".format(key))
|
||||
continue
|
||||
new = HTTPS_BASE + path
|
||||
if old != new:
|
||||
changes.append((modules_file, key, old, new))
|
||||
if key not in local:
|
||||
changes.append((config_file, key, None, SSH_BASE + path))
|
||||
for key, old in local.items():
|
||||
path = repository_path(old)
|
||||
if path is None:
|
||||
print("Skipping unrecognized local URL: {}".format(key))
|
||||
continue
|
||||
new = SSH_BASE + path
|
||||
if old != new:
|
||||
changes.append((config_file, key, old, new))
|
||||
for file, key, old, new in changes:
|
||||
print("{}: {}\n {} -> {}".format(file, key, old or "(missing)", new))
|
||||
if not dry_run:
|
||||
git("config", "--file", str(file), "--replace-all", key, new)
|
||||
print("{} {} URL(s).".format("Would update" if dry_run else "Updated", len(changes)))
|
||||
return len(changes)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("repository", nargs="?", default=".",
|
||||
help="repository directory (default: current directory)")
|
||||
parser.add_argument("--dry-run", action="store_true",
|
||||
help="show planned changes without writing files")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
fix(args.repository, args.dry_run)
|
||||
except (OSError, RuntimeError, ValueError) as error:
|
||||
print("Error: {}".format(error), file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
"""Opt-in network test; run with unittest discover -s integration -v."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from fix_submodules import git, read_urls, repository_path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[1] / 'fix_submodules.py'
|
||||
CLONE_URL = 'https://proxy.dandaedre.com/gitea/Ravenous/ravenous_roll20.git'
|
||||
|
||||
|
||||
class CloneTest(unittest.TestCase):
|
||||
def test_real_repository(self):
|
||||
with tempfile.TemporaryDirectory(prefix='repo-fix-ci-') as directory:
|
||||
root = Path(directory) / 'ravenous'
|
||||
subprocess.run(
|
||||
['git', 'clone', '--depth', '1', '--no-recurse-submodules', CLONE_URL, str(root)],
|
||||
env=dict(os.environ, GIT_TERMINAL_PROMPT='0'), check=True, timeout=120,
|
||||
)
|
||||
modules = root / '.gitmodules'
|
||||
config = root / '.git/config'
|
||||
original = read_urls(modules)
|
||||
paths = {key: repository_path(url) for key, url in original.items()
|
||||
if repository_path(url) is not None}
|
||||
self.assertTrue(paths, 'Expected at least one Dandaedre submodule to exercise')
|
||||
|
||||
def run_fix(*args):
|
||||
subprocess.run([sys.executable, str(SCRIPT), str(root), *args],
|
||||
check=True, timeout=30)
|
||||
|
||||
def snapshot():
|
||||
return modules.read_bytes(), config.read_bytes()
|
||||
|
||||
def check_urls():
|
||||
public, local = read_urls(modules), read_urls(config)
|
||||
for key, path in paths.items():
|
||||
self.assertEqual(public[key], 'https://proxy.dandaedre.com/gitea/' + path)
|
||||
self.assertEqual(local[key], 'git@git.dandaedre.com:' + path)
|
||||
for key, url in original.items():
|
||||
if key not in paths:
|
||||
self.assertEqual(public[key], url)
|
||||
|
||||
before = snapshot()
|
||||
run_fix('--dry-run')
|
||||
self.assertEqual(snapshot(), before, 'Dry run changed files')
|
||||
run_fix()
|
||||
check_urls()
|
||||
fixed = snapshot()
|
||||
run_fix()
|
||||
self.assertEqual(snapshot(), fixed, 'Second run changed files')
|
||||
|
||||
# Reproduce Git overwriting local URLs with the CI HTTPS URLs.
|
||||
git('submodule', 'init', cwd=root)
|
||||
git('submodule', 'sync', cwd=root)
|
||||
for key, path in paths.items():
|
||||
self.assertEqual(read_urls(config)[key],
|
||||
'https://proxy.dandaedre.com/gitea/' + path)
|
||||
run_fix()
|
||||
check_urls()
|
||||
|
||||
# Exercise legacy hostnames using real submodule names and paths.
|
||||
for file in (modules, config):
|
||||
for key, path in paths.items():
|
||||
git('config', '--file', str(file), key, 'git@dandaedre.com:' + path)
|
||||
run_fix()
|
||||
check_urls()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
import contextlib
|
||||
import io
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from fix_submodules import fix, git, read_urls, repository_path
|
||||
|
||||
|
||||
class FixTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(directory.cleanup)
|
||||
self.root = Path(directory.name)
|
||||
git('init', str(self.root))
|
||||
self.modules = self.root / '.gitmodules'
|
||||
self.config = self.root / '.git/config'
|
||||
self.modules.write_text(
|
||||
'# Keep this comment\n[submodule "reportist"]\n'
|
||||
'\tpath = reportist\n'
|
||||
'\turl = git@dandaedre.com:Beffroi/reportist.git\n'
|
||||
'[submodule "other"]\n\tpath = other\n'
|
||||
'\turl = https://example.com/owner/other.git\n'
|
||||
)
|
||||
git('config', '--file', str(self.config), 'submodule.reportist.active', 'true')
|
||||
|
||||
def run_fix(self, **kwargs):
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
return fix(self.root, **kwargs)
|
||||
|
||||
def test_dry_run_apply_and_repeat(self):
|
||||
before = (self.modules.read_bytes(), self.config.read_bytes())
|
||||
self.assertEqual(self.run_fix(dry_run=True), 2)
|
||||
self.assertEqual(before, (self.modules.read_bytes(), self.config.read_bytes()))
|
||||
self.assertEqual(self.run_fix(), 2)
|
||||
self.assertEqual(read_urls(self.modules)['submodule.reportist.url'],
|
||||
'https://proxy.dandaedre.com/gitea/Beffroi/reportist.git')
|
||||
self.assertEqual(read_urls(self.config)['submodule.reportist.url'],
|
||||
'git@git.dandaedre.com:Beffroi/reportist.git')
|
||||
self.assertIn('# Keep this comment', self.modules.read_text())
|
||||
self.assertIn('active = true', self.config.read_text())
|
||||
self.assertEqual(read_urls(self.modules)['submodule.other.url'],
|
||||
'https://example.com/owner/other.git')
|
||||
self.assertEqual(self.run_fix(), 0)
|
||||
|
||||
def test_existing_local_path(self):
|
||||
for url in ('https://proxy.dandaedre.com/gitea/Beffroi/fork.git',
|
||||
'git@dandaedre.com:Beffroi/fork.git'):
|
||||
with self.subTest(url=url):
|
||||
git('config', '--file', str(self.config), 'submodule.reportist.url', url)
|
||||
self.run_fix()
|
||||
self.assertEqual(read_urls(self.config)['submodule.reportist.url'],
|
||||
'git@git.dandaedre.com:Beffroi/fork.git')
|
||||
|
||||
def test_missing_gitmodules(self):
|
||||
self.modules.unlink()
|
||||
self.assertEqual(self.run_fix(), 0)
|
||||
|
||||
def test_linked_worktree(self):
|
||||
git('-c', 'user.name=Test', '-c', 'user.email=test@example.com',
|
||||
'commit', '--allow-empty', '-m', 'initial', cwd=self.root)
|
||||
linked = self.root / 'linked'
|
||||
git('worktree', 'add', '-b', 'linked', str(linked), cwd=self.root)
|
||||
(linked / '.gitmodules').write_bytes(self.modules.read_bytes())
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
self.assertEqual(fix(linked), 2)
|
||||
self.assertEqual(read_urls(self.config)['submodule.reportist.url'],
|
||||
'git@git.dandaedre.com:Beffroi/reportist.git')
|
||||
|
||||
def test_url_forms(self):
|
||||
for url in ('git@git.dandaedre.com:Beffroi/reportist.git',
|
||||
'ssh://git@dandaedre.com/Beffroi/reportist.git',
|
||||
'https://proxy.dandaedre.com/gitea/Beffroi/reportist.git',
|
||||
'git://git.dandaedre.com/Beffroi/reportist.git'):
|
||||
self.assertEqual(repository_path(url), 'Beffroi/reportist.git')
|
||||
for url in ('../other.git', 'git@example.com:owner/repo.git',
|
||||
'https://dandaedre.com.evil.example/owner/repo.git'):
|
||||
self.assertIsNone(repository_path(url))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue