diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..86b5c16 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,49 @@ +// Optional Gitea webhook: http://192.168.1.37:8080/gitea-webhook/post +pipeline { + agent { label 'rust_no_docker' } + + options { + skipDefaultCheckout(true) + timeout(time: 10, unit: 'MINUTES') + disableConcurrentBuilds() + } + + parameters { + booleanParam(name: 'RUN_CLONE_TEST', defaultValue: true, + description: 'Test URL repair on a temporary Ravenous clone (requires Gitea access).') + } + + environment { + GIT_TERMINAL_PROMPT = '0' + PYTHONUNBUFFERED = '1' + } + + stages { + stage('Checkout') { + steps { + checkout scm + } + } + stage('Check tools') { + steps { + sh 'python3 --version && git --version' + } + } + stage('Unit tests') { + steps { + sh 'python3 -m unittest discover -s tests -v' + } + } + stage('Ravenous clone test') { + when { expression { params.RUN_CLONE_TEST } } + steps { + sh 'python3 -m unittest discover -s integration -v' + } + } + } + + post { + success { echo 'repo-fix tests passed.' } + failure { echo 'repo-fix tests failed; see the test output above.' } + } +} diff --git a/README.md b/README.md index 91a715c..5e412a8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,78 @@ # repo-fix -Fix repository submodules for the CI \ No newline at end of file +Fix Dandaedre Gitea submodule URLs after cloning, pulling, or syncing: + +- `.gitmodules` uses HTTPS for CI: `https://proxy.dandaedre.com/gitea/Beffroi/reportist.git` +- `.git/config` uses SSH locally: `git@git.dandaedre.com:Beffroi/reportist.git` + +Requires Python 3 and Git on your PATH. No pip packages are needed; the script +uses the Python standard library and Git's config commands. + +Preview the changes, then apply them: + +```bash +python3 fix_submodules.py /path/to/repository --dry-run +python3 fix_submodules.py /path/to/repository +``` + +The repository defaults to the current directory, and may also be a directory +inside a working tree. Run the script again whenever a pull or `git submodule +sync` changes the URLs. Repeated runs are safe. Dry runs do not write files. + +Recognizes HTTP(S), SSH, Git protocol, and SCP-style URLs on `dandaedre.com`, +`git.dandaedre.com`, and `proxy.dandaedre.com`. Old URLs such as +`git@dandaedre.com:Beffroi/reportist.git` are corrected. Repository paths and +`.git` suffixes are preserved; the proxy's `/gitea/` prefix is translated. + +Missing local URL entries are added from `.gitmodules`. Existing local URLs +retain their own repository paths. Other hosts and relative URLs are skipped. +Unrelated configuration, including `active`, is preserved. + +Works with `.git` directories and pointer files, including linked worktrees +(whose local config is shared). Only the selected repository's submodule +entries are changed. The script does not recurse, change `origin` remotes, or +fetch anything. Run it separately inside submodules with their own declarations. + +Commit `.gitmodules` changes for CI. Local config changes are not committed. +After fixing URLs, initialize submodules with: + +```bash +git -C /path/to/repository submodule update --init --recursive +``` + +Run tests: + +```bash +python3 -m unittest discover -s tests -v +``` + +## Jenkins test CI + +The `Jenkinsfile` uses the `rust_no_docker` agent from the Ravenous pipeline. +Configure a Jenkins Pipeline from SCM (or a multibranch pipeline) for this +repository with script path `Jenkinsfile`. The agent needs `python3` and `git` +on PATH and network access to the Gitea HTTPS proxy. No uv, Docker, or package +installation is required. Use the Gitea webhook URL from the Jenkinsfile if +your Jenkins Gitea integration is configured at that address. + +Every build runs the unit tests. By default, it also shallow-clones +`Ravenous/ravenous_roll20.git` through the HTTPS proxy into a temporary directory +and verifies dry-run safety, both URL formats, repeatability, recovery after +`git submodule sync`, and correction of the old SSH hostname. The clone is +removed after the test. It does not fetch submodules or run code from Ravenous; +SSH connectivity and recursive submodule downloads are not tested. + +Run the same network test locally: + +```bash +python3 -m unittest discover -s integration -v +``` + +The HTTPS clone URL corresponds to +`git@dandaedre.com:Ravenous/ravenous_roll20.git` and avoids requiring an SSH key +for this CI test. If authentication is required, configure Git credentials on +the agent; prompts are disabled. Clone failures fail the build. Disable the +`RUN_CLONE_TEST` build parameter for an offline unit-test-only run. + +This pipeline only checks out and tests. It does not publish artifacts, send +checks, push commits, or deploy files. Test results appear in the console log. diff --git a/fix_submodules.py b/fix_submodules.py new file mode 100644 index 0000000..e2d1b68 --- /dev/null +++ b/fix_submodules.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Normalize Dandaedre submodule URLs for CI and local SSH access.""" + +import argparse +from pathlib import Path +import re +import subprocess +import sys +from urllib.parse import urlsplit + + +HTTPS_BASE = "https://proxy.dandaedre.com/gitea/" +SSH_BASE = "git@git.dandaedre.com:" +HOSTS = {"dandaedre.com", "git.dandaedre.com", "proxy.dandaedre.com"} + + +def repository_path(url): + """Return the repository path for a recognized absolute URL, or None.""" + if "://" in url: + parsed = urlsplit(url) + if parsed.scheme not in {"http", "https", "ssh", "git"}: + return None + host = parsed.hostname + if host not in HOSTS or parsed.query or parsed.fragment: + return None + path = parsed.path.lstrip("/") + else: + match = re.fullmatch(r"(?:[^@/:]+@)?([^/:]+):(.+)", url) + if not match or match[1].lower() not in HOSTS: + return None + host, path = match[1].lower(), match[2].lstrip("/") + if host == "proxy.dandaedre.com" and path.startswith("gitea/"): + path = path[len("gitea/"):] + if len(path.split("/")) < 2 or any(p in {"", ".", ".."} for p in path.split("/")): + return None + return path + + +def git(*args, cwd=None, allow_missing=False): + result = subprocess.run( + ["git", *args], cwd=cwd, text=True, capture_output=True, check=False + ) + if result.returncode and not (allow_missing and result.returncode == 1): + raise RuntimeError(result.stderr.strip() or "Git command failed") + return result.stdout + + +def read_urls(path): + if not path.exists(): + return {} + output = git("config", "--file", str(path), "--null", "--get-regexp", + r"^submodule\..*\.url$", allow_missing=True) + urls = {} + for record in output.split("\0"): + if record: + key, value = record.split("\n", 1) + if key in urls: + raise RuntimeError("Duplicate submodule URL key in {}: {}".format(path, key)) + urls[key] = value + return urls + + +def fix(repository, dry_run=False): + root = Path(git("rev-parse", "--show-toplevel", cwd=repository).strip()) + common = Path(git("rev-parse", "--git-common-dir", cwd=root).strip()) + if not common.is_absolute(): + common = root / common + modules_file = root / ".gitmodules" + config_file = common.resolve() / "config" + modules = read_urls(modules_file) + local = read_urls(config_file) + changes = [] + for key, old in modules.items(): + path = repository_path(old) + if path is None: + print("Skipping unrecognized URL in .gitmodules: {}".format(key)) + continue + new = HTTPS_BASE + path + if old != new: + changes.append((modules_file, key, old, new)) + if key not in local: + changes.append((config_file, key, None, SSH_BASE + path)) + for key, old in local.items(): + path = repository_path(old) + if path is None: + print("Skipping unrecognized local URL: {}".format(key)) + continue + new = SSH_BASE + path + if old != new: + changes.append((config_file, key, old, new)) + for file, key, old, new in changes: + print("{}: {}\n {} -> {}".format(file, key, old or "(missing)", new)) + if not dry_run: + git("config", "--file", str(file), "--replace-all", key, new) + print("{} {} URL(s).".format("Would update" if dry_run else "Updated", len(changes))) + return len(changes) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("repository", nargs="?", default=".", + help="repository directory (default: current directory)") + parser.add_argument("--dry-run", action="store_true", + help="show planned changes without writing files") + args = parser.parse_args() + try: + fix(args.repository, args.dry_run) + except (OSError, RuntimeError, ValueError) as error: + print("Error: {}".format(error), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integration/test_clone.py b/integration/test_clone.py new file mode 100644 index 0000000..6043db8 --- /dev/null +++ b/integration/test_clone.py @@ -0,0 +1,75 @@ +"""Opt-in network test; run with unittest discover -s integration -v.""" + +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from fix_submodules import git, read_urls, repository_path + + +SCRIPT = Path(__file__).resolve().parents[1] / 'fix_submodules.py' +CLONE_URL = 'https://proxy.dandaedre.com/gitea/Ravenous/ravenous_roll20.git' + + +class CloneTest(unittest.TestCase): + def test_real_repository(self): + with tempfile.TemporaryDirectory(prefix='repo-fix-ci-') as directory: + root = Path(directory) / 'ravenous' + subprocess.run( + ['git', 'clone', '--depth', '1', '--no-recurse-submodules', CLONE_URL, str(root)], + env=dict(os.environ, GIT_TERMINAL_PROMPT='0'), check=True, timeout=120, + ) + modules = root / '.gitmodules' + config = root / '.git/config' + original = read_urls(modules) + paths = {key: repository_path(url) for key, url in original.items() + if repository_path(url) is not None} + self.assertTrue(paths, 'Expected at least one Dandaedre submodule to exercise') + + def run_fix(*args): + subprocess.run([sys.executable, str(SCRIPT), str(root), *args], + check=True, timeout=30) + + def snapshot(): + return modules.read_bytes(), config.read_bytes() + + def check_urls(): + public, local = read_urls(modules), read_urls(config) + for key, path in paths.items(): + self.assertEqual(public[key], 'https://proxy.dandaedre.com/gitea/' + path) + self.assertEqual(local[key], 'git@git.dandaedre.com:' + path) + for key, url in original.items(): + if key not in paths: + self.assertEqual(public[key], url) + + before = snapshot() + run_fix('--dry-run') + self.assertEqual(snapshot(), before, 'Dry run changed files') + run_fix() + check_urls() + fixed = snapshot() + run_fix() + self.assertEqual(snapshot(), fixed, 'Second run changed files') + + # Reproduce Git overwriting local URLs with the CI HTTPS URLs. + git('submodule', 'init', cwd=root) + git('submodule', 'sync', cwd=root) + for key, path in paths.items(): + self.assertEqual(read_urls(config)[key], + 'https://proxy.dandaedre.com/gitea/' + path) + run_fix() + check_urls() + + # Exercise legacy hostnames using real submodule names and paths. + for file in (modules, config): + for key, path in paths.items(): + git('config', '--file', str(file), key, 'git@dandaedre.com:' + path) + run_fix() + check_urls() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_fix_submodules.py b/tests/test_fix_submodules.py new file mode 100644 index 0000000..e3dfb8d --- /dev/null +++ b/tests/test_fix_submodules.py @@ -0,0 +1,82 @@ +import contextlib +import io +from pathlib import Path +import tempfile +import unittest + +from fix_submodules import fix, git, read_urls, repository_path + + +class FixTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + git('init', str(self.root)) + self.modules = self.root / '.gitmodules' + self.config = self.root / '.git/config' + self.modules.write_text( + '# Keep this comment\n[submodule "reportist"]\n' + '\tpath = reportist\n' + '\turl = git@dandaedre.com:Beffroi/reportist.git\n' + '[submodule "other"]\n\tpath = other\n' + '\turl = https://example.com/owner/other.git\n' + ) + git('config', '--file', str(self.config), 'submodule.reportist.active', 'true') + + def run_fix(self, **kwargs): + with contextlib.redirect_stdout(io.StringIO()): + return fix(self.root, **kwargs) + + def test_dry_run_apply_and_repeat(self): + before = (self.modules.read_bytes(), self.config.read_bytes()) + self.assertEqual(self.run_fix(dry_run=True), 2) + self.assertEqual(before, (self.modules.read_bytes(), self.config.read_bytes())) + self.assertEqual(self.run_fix(), 2) + self.assertEqual(read_urls(self.modules)['submodule.reportist.url'], + 'https://proxy.dandaedre.com/gitea/Beffroi/reportist.git') + self.assertEqual(read_urls(self.config)['submodule.reportist.url'], + 'git@git.dandaedre.com:Beffroi/reportist.git') + self.assertIn('# Keep this comment', self.modules.read_text()) + self.assertIn('active = true', self.config.read_text()) + self.assertEqual(read_urls(self.modules)['submodule.other.url'], + 'https://example.com/owner/other.git') + self.assertEqual(self.run_fix(), 0) + + def test_existing_local_path(self): + for url in ('https://proxy.dandaedre.com/gitea/Beffroi/fork.git', + 'git@dandaedre.com:Beffroi/fork.git'): + with self.subTest(url=url): + git('config', '--file', str(self.config), 'submodule.reportist.url', url) + self.run_fix() + self.assertEqual(read_urls(self.config)['submodule.reportist.url'], + 'git@git.dandaedre.com:Beffroi/fork.git') + + def test_missing_gitmodules(self): + self.modules.unlink() + self.assertEqual(self.run_fix(), 0) + + def test_linked_worktree(self): + git('-c', 'user.name=Test', '-c', 'user.email=test@example.com', + 'commit', '--allow-empty', '-m', 'initial', cwd=self.root) + linked = self.root / 'linked' + git('worktree', 'add', '-b', 'linked', str(linked), cwd=self.root) + (linked / '.gitmodules').write_bytes(self.modules.read_bytes()) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(fix(linked), 2) + self.assertEqual(read_urls(self.config)['submodule.reportist.url'], + 'git@git.dandaedre.com:Beffroi/reportist.git') + + def test_url_forms(self): + for url in ('git@git.dandaedre.com:Beffroi/reportist.git', + 'ssh://git@dandaedre.com/Beffroi/reportist.git', + 'https://proxy.dandaedre.com/gitea/Beffroi/reportist.git', + 'git://git.dandaedre.com/Beffroi/reportist.git'): + self.assertEqual(repository_path(url), 'Beffroi/reportist.git') + for url in ('../other.git', 'git@example.com:owner/repo.git', + 'https://dandaedre.com.evil.example/owner/repo.git'): + self.assertIsNone(repository_path(url)) + + +if __name__ == '__main__': + unittest.main()