This site hosts my projects.
commit 5d3121e5485902cb6ef59e3a6bd6fcef8fa2f5f7
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Mon May 5 21:05:19 2025 +0200
Extract RsyncCommand.
diff --git a/projects2.py b/projects2.py
index 347eea9..0ab01ad 100755
--- a/projects2.py
+++ b/projects2.py
@@ -424,14 +424,13 @@ class Projects2:
>>> events
RUN => ['rsync', '--archive', '--delete', '--verbose', 'scm@projects.rickardlindberg.me:/opt/rlprojects/', 'local.bak']
"""
- self.process.ensure([
- "rsync",
- "--archive",
- "--delete",
- "--verbose",
- f"{self.config.SSH_USER}@{server}:{ensure_ends_with_slash(self.config.INSTANCE_ROOT)}",
- ensure_has_no_slash_at_end(local_folder)
- ], capture=False)
+ self.process.ensure(
+ RsyncCommand().sync_folders(
+ source=f"{self.config.SSH_USER}@{server}:{self.config.INSTANCE_ROOT}",
+ destination=local_folder,
+ ),
+ capture=False,
+ )
def restore(self, server, local_folder):
"""
@@ -443,14 +442,13 @@ class Projects2:
>>> events
RUN => ['rsync', '--archive', '--delete', '--verbose', 'local.bak/', 'scm@projects.rickardlindberg.me:/opt/rlprojects']
"""
- self.process.ensure([
- "rsync",
- "--archive",
- "--delete",
- "--verbose",
- ensure_ends_with_slash(local_folder),
- f"{self.config.SSH_USER}@{server}:{ensure_has_no_slash_at_end(self.config.INSTANCE_ROOT)}",
- ], capture=False)
+ self.process.ensure(
+ RsyncCommand().sync_folders(
+ source=local_folder,
+ destination=f"{self.config.SSH_USER}@{server}:{self.config.INSTANCE_ROOT}",
+ ),
+ capture=False,
+ )
def upload_artifact(self, server, project, source, destination):
"""
@@ -1410,16 +1408,11 @@ server {{
user=self.config.SSH_USER,
group=self.config.SSH_USER,
)
- self.process.ensure([
- "rsync",
- "--archive",
- "--checksum",
- "--no-times",
- "--delete",
- "--verbose",
- ensure_ends_with_slash(join(repo_path, files_json["site"])),
- ensure_has_no_slash_at_end(site_root),
- ])
+ self.process.ensure(RsyncCommand().sync_folders(
+ source=join(repo_path, files_json["site"]),
+ destination=site_root,
+ skip_timestamps=True,
+ ))
else:
self.fedora_system.ensure_gone(site_root)
for artifact in files_json["artifacts"]:
@@ -2650,16 +2643,38 @@ def replace_lines(text, replacements):
new.append(replacement)
return "\n".join(new+[""])
-def ensure_ends_with_slash(path):
- if path.endswith("/"):
- return path
- else:
- return path + "/"
+class RsyncCommand:
-def ensure_has_no_slash_at_end(path):
- while path.endswith("/"):
- path = path[:-1]
- return path
+ def sync_folders(self, source, destination, skip_timestamps=False):
+ return [
+ "rsync",
+ "--archive",
+ ]+self.skip_timestamps_part(skip_timestamps)+[
+ "--delete",
+ "--verbose",
+ self.ensure_ends_with_slash(source),
+ self.ensure_has_no_slash_at_end(destination),
+ ]
+
+ def skip_timestamps_part(self, skip_timestamps):
+ if skip_timestamps:
+ return [
+ "--checksum",
+ "--no-times",
+ ]
+ else:
+ return []
+
+ def ensure_ends_with_slash(self, path):
+ if path.endswith("/"):
+ return path
+ else:
+ return path + "/"
+
+ def ensure_has_no_slash_at_end(self, path):
+ while path.endswith("/"):
+ path = path[:-1]
+ return path
def extract_section(section, config):
"""
commit be619372525ebfd6d1a3c8dda0f076bf71d66d1c
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Mon May 5 08:17:58 2025 +0200
Push functionality from Project to Repo.
diff --git a/projects2.py b/projects2.py
index 9e29f12..347eea9 100755
--- a/projects2.py
+++ b/projects2.py
@@ -161,9 +161,6 @@ class Config:
def git_repo_path(self, project):
return join(self.SCM_ROOT, f"{project}.git")
- def hg_repo_path_hg(self, project):
- return join(self.hg_repo_path(project), ".hg")
-
def hg_repo_path(self, project):
return join(self.SCM_ROOT, project)
@@ -2496,16 +2493,7 @@ class Project:
if latest_event is not None:
return latest_event.date
else:
- if self.filesystem.exists(self.git_repo_path):
- return self.filesystem.get_modification_time(
- self.git_repo_path,
- )
- elif self.filesystem.exists(self.hg_repo_path_hg):
- return self.filesystem.get_modification_time(
- self.hg_repo_path_hg
- )
- else:
- return datetime.fromtimestamp(0).astimezone(timezone.utc)
+ return self.repo.modification_time
def diff(self, old, new):
return self.repo.diff(old, new)
@@ -2513,23 +2501,20 @@ class Project:
@property
def repo(self):
if self.config.is_hg(self.name):
- return Hg(self.process, self.filesystem, self.fedora_system, self.hg_repo_path, self.config)
+ return Hg(self.process, self.filesystem, self.fedora_system, self.config.hg_repo_path(self.name), self.config)
else:
- return Git(self.process, self.filesystem, self.fedora_system, self.git_repo_path, self.config)
+ return Git(self.process, self.filesystem, self.fedora_system, self.config.git_repo_path(self.name), self.config)
- @property
- def git_repo_path(self):
- return self.config.git_repo_path(self.name)
-
- @property
- def hg_repo_path_hg(self):
- return self.config.hg_repo_path_hg(self.name)
+class Repo:
@property
- def hg_repo_path(self):
- return self.config.hg_repo_path(self.name)
+ def modification_time(self):
+ if self.filesystem.exists(self.path):
+ return self.filesystem.get_modification_time(self.path)
+ else:
+ return datetime.fromtimestamp(0).astimezone(timezone.utc)
-class Git:
+class Git(Repo):
def __init__(self, process, filesystem, fedora_system, path, config):
self.process = process
@@ -2588,7 +2573,7 @@ exec git update-server-info
group=self.config.SSH_USER,
)
-class Hg:
+class Hg(Repo):
def __init__(self, process, filesystem, fedora_system, path, config):
self.process = process
commit 075a93c4004be34659fe8f4ac44b223cb91b2246
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sun May 4 22:06:47 2025 +0200
Ensure scm is part of config and not hard coded in the rest of the code.
diff --git a/projects2.py b/projects2.py
index 9606c6e..9e29f12 100755
--- a/projects2.py
+++ b/projects2.py
@@ -138,6 +138,13 @@ class Config:
self.PROJECT_NAME_ENV_VAR = "PROJECT_NAME"
self.USER_ID_ENV_VAR = "USER_ID"
+ def is_hg(self, project):
+ return self.PROJECTS.get(project, {}).get("scm", "git") == "hg"
+
+ def scm_root_url(self):
+ part = relpath(self.SCM_ROOT, start=self.WEB_ROOT)
+ return f"{self.DOMAIN}/{part}"
+
def get_site_root(self, project):
return join(self.WEB_ROOT, project)
@@ -151,6 +158,15 @@ class Config:
def get_list(self, section, name):
return [x.strip() for x in self.config.get(section, name).split(",")]
+ def git_repo_path(self, project):
+ return join(self.SCM_ROOT, f"{project}.git")
+
+ def hg_repo_path_hg(self, project):
+ return join(self.hg_repo_path(project), ".hg")
+
+ def hg_repo_path(self, project):
+ return join(self.SCM_ROOT, project)
+
@property
def wildcard_certificate(self):
"""
@@ -1187,10 +1203,10 @@ server {{
>>> print(Projects2.create_null().clone_instructions("projects2"))
git clone https://projects.rickardlindberg.me/scm/projects2.git
"""
- if self.config.PROJECTS.get(project, {}).get("scm", "git") == "hg":
- return f"hg clone static-http://{self.config.DOMAIN}/scm/{project}"
+ if self.config.is_hg(project):
+ return f"hg clone static-http://{self.config.scm_root_url()}/{project}"
else:
- return f"git clone https://{self.config.DOMAIN}/scm/{project}.git"
+ return f"git clone https://{self.config.scm_root_url()}/{project}.git"
def pre_receive(self):
"""
@@ -1564,21 +1580,21 @@ server {{
sys.exit("Permission denied.")
self.env.set(self.config.PROJECT_NAME_ENV_VAR, project)
self.env.set(self.config.USER_ID_ENV_VAR, user_id)
- self.process.exec(["git-receive-pack", join(self.config.WEB_ROOT, "scm", f"{project}.git")])
+ self.process.exec(["git-receive-pack", join(self.config.SCM_ROOT, f"{project}.git")])
elif original == ["git-upload-pack", Endswith(".git")]:
project = original[1].split(".")[0]
if not self.is_member_of_project(project, memberships):
sys.exit("Permission denied.")
self.env.set(self.config.PROJECT_NAME_ENV_VAR, project)
self.env.set(self.config.USER_ID_ENV_VAR, user_id)
- self.process.exec(["git-upload-pack", join(self.config.WEB_ROOT, "scm", f"{project}.git")])
+ self.process.exec(["git-upload-pack", join(self.config.SCM_ROOT, f"{project}.git")])
elif original == ["hg", "-R", Any(), "serve", "--stdio"]:
project = original[2]
if not self.is_member_of_project(project, memberships):
sys.exit("Permission denied.")
self.env.set(self.config.PROJECT_NAME_ENV_VAR, project)
self.env.set(self.config.USER_ID_ENV_VAR, user_id)
- self.process.exec(original[:2]+[join(self.config.WEB_ROOT, "scm", project)]+original[3:])
+ self.process.exec(original[:2]+[join(self.config.SCM_ROOT, project)]+original[3:])
elif original == [self.config.INSTANCE_SCRIPT, "upload-artifact-server", Any(), Any(), Any()]:
project = original[2]
if not self.is_member_of_project(project, memberships):
@@ -2496,22 +2512,22 @@ class Project:
@property
def repo(self):
- if self.config.PROJECTS.get(self.name, {}).get("scm", "git") == "hg":
+ if self.config.is_hg(self.name):
return Hg(self.process, self.filesystem, self.fedora_system, self.hg_repo_path, self.config)
else:
return Git(self.process, self.filesystem, self.fedora_system, self.git_repo_path, self.config)
@property
def git_repo_path(self):
- return join(self.config.WEB_ROOT, "scm", f"{self.name}.git")
+ return self.config.git_repo_path(self.name)
@property
def hg_repo_path_hg(self):
- return join(self.hg_repo_path, ".hg")
+ return self.config.hg_repo_path_hg(self.name)
@property
def hg_repo_path(self):
- return join(self.config.WEB_ROOT, "scm", self.name)
+ return self.config.hg_repo_path(self.name)
class Git:
@@ -2548,7 +2564,7 @@ class Git:
"--initial-branch", "main",
self.path
])
- self.process.run(["chown", "-R", "scm:scm", self.path])
+ self.process.run(["chown", "-R", f"{self.config.SSH_USER}:{self.config.SSH_USER}", self.path])
self.fedora_system.ensure_file(
description="pre-receive hook",
name=join(self.path, "hooks/pre-receive"),
@@ -2605,7 +2621,7 @@ class Hg:
"hg", "init",
self.path
])
- self.process.run(["chown", "-R", "scm:scm", self.path])
+ self.process.run(["chown", "-R", f"{self.config.SSH_USER}:{self.config.SSH_USER}", self.path])
self.fedora_system.ensure_file(
description="hgrc",
name=join(self.path, ".hg", "hgrc"),
commit d437e70a575d716c480dca5e197cb47cf9663ed6
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sun May 4 21:50:45 2025 +0200
Use generic Events.replace for tab handling.
diff --git a/projects2.py b/projects2.py
index f10e66d..9606c6e 100755
--- a/projects2.py
+++ b/projects2.py
@@ -582,7 +582,7 @@ class Projects2:
... },
... events=events,
... ).run()
- >>> events
+ >>> events.replace("\\t", "<tab>")
ENSURE_USER =>
name: 'scm'
ENSURE_LINES_IN_FILE =>
@@ -2350,7 +2350,7 @@ class Events:
return [replace(x) for x in value]
elif isinstance(value, dict):
return {
- key: replace(dict_value)
+ replace(key): replace(dict_value)
for key, dict_value in value.items()
}
else:
@@ -2365,14 +2365,12 @@ class Events:
return self
def __repr__(self):
- def replace_tab(text):
- return text.replace("\t", "<tab>")
def strip_line(line):
- return replace_tab(line.rstrip())
+ return line.rstrip()
def repr_value(value, indent=1):
if isinstance(value, dict):
return "".join([
- f"\n{' '*indent}{replace_tab(key)}:{repr_value(subvalue, indent+1)}"
+ f"\n{' '*indent}{key}:{repr_value(subvalue, indent+1)}"
for key, subvalue in value.items()
])
elif isinstance(value, str) and "\n" in value:
@@ -2381,7 +2379,7 @@ class Events:
for line in value.splitlines()
])
elif isinstance(value, str):
- return f" {repr(replace_tab(value))}"
+ return f" {repr(value)}"
else:
return f" {value!r}"
return "\n".join(
commit 9aa4dbb4aebc29cf43264ccbb20cf1b9f1c8843e
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sun May 4 21:47:13 2025 +0200
Get rid of ELLIPSIS by converting ProcessResult to an event and using replace.
diff --git a/projects2.py b/projects2.py
index fd13409..f10e66d 100755
--- a/projects2.py
+++ b/projects2.py
@@ -2104,18 +2104,33 @@ class Filesystem:
class Process:
"""
- >>> Process.create().run([sys.executable, "-c", "import sys; sys.stdout.write('out'); sys.stderr.write('err'); sys.exit(3)"]) # doctest: +ELLIPSIS
- ProcessResult(command=['...', '-c', "import sys; sys.stdout.write('out'); sys.stderr.write('err'); sys.exit(3)"], stdout='out', stderr='err', returncode=3)
+ >>> Process.create().run([
+ ... sys.executable, "-c",
+ ... "import sys; sys.stdout.write('out'); sys.stderr.write('err'); sys.exit(3)"
+ ... ]).as_events().replace(sys.executable, "<EXECUTABLE>")
+ PROCESS_RESULT =>
+ command: ['<EXECUTABLE>', '-c', "import sys; sys.stdout.write('out'); sys.stderr.write('err'); sys.exit(3)"]
+ stdout: 'out'
+ stderr: 'err'
+ returncode: 3
>>> Process.create_null({
... ("my-test-script",): {"stdout": "hello", "stderr": "world", "returncode": 4},
- ... }).run(["my-test-script"])
- ProcessResult(command=['my-test-script'], stdout='hello', stderr='world', returncode=4)
+ ... }).run(["my-test-script"]).as_events()
+ PROCESS_RESULT =>
+ command: ['my-test-script']
+ stdout: 'hello'
+ stderr: 'world'
+ returncode: 4
>>> Process.create_null({
... ("other",): {"returncode": 4},
- ... }).run(["my-test-script"])
- ProcessResult(command=['my-test-script'], stdout='', stderr='', returncode=0)
+ ... }).run(["my-test-script"]).as_events()
+ PROCESS_RESULT =>
+ command: ['my-test-script']
+ stdout: ''
+ stderr: ''
+ returncode: 0
"""
@classmethod
@@ -2211,15 +2226,13 @@ Command failed:
""")
return self
- def __repr__(self):
- return (
- "ProcessResult("
- f"command={self.command!r}, "
- f"stdout={self.stdout!r}, "
- f"stderr={self.stderr!r}, "
- f"returncode={self.returncode!r}"
- ")"
- )
+ def as_events(self):
+ return Events().add("PROCESS_RESULT", {
+ "command": self.command,
+ "stdout": self.stdout,
+ "stderr": self.stderr,
+ "returncode": self.returncode,
+ })
@Trackable
class Env:
@@ -2333,6 +2346,8 @@ class Events:
def replace(value):
if isinstance(value, str):
return value.replace(thing, placeholder)
+ elif isinstance(value, list):
+ return [replace(x) for x in value]
elif isinstance(value, dict):
return {
key: replace(dict_value)
@@ -2347,6 +2362,7 @@ class Events:
def add(self, name, value):
self.events.append((name, value))
+ return self
def __repr__(self):
def replace_tab(text):
commit 78b0e78008e7ba3722c35a2140a0834b490c6faf
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sun May 4 21:40:49 2025 +0200
Allow taking the first event only and asserting on that.
diff --git a/projects2.py b/projects2.py
index c3c80ef..fd13409 100755
--- a/projects2.py
+++ b/projects2.py
@@ -1609,8 +1609,8 @@ server {{
"""
>>> events = Events()
>>> Projects2.create_null(args=[], events=events).run()
- >>> events # doctest: +ELLIPSIS
- LOG => 'Initial Setup...
+ >>> events.take(1)
+ LOG => 'Initial Setup:'
"""
self.logger.log("Initial Setup:")
self.logger.log("")
@@ -2326,6 +2326,9 @@ class Events:
def __init__(self, events=None):
self.events = [] if events is None else events
+ def take(self, n=1):
+ return Events(self.events[:n])
+
def replace(self, thing, placeholder):
def replace(value):
if isinstance(value, str):
commit 819fd507c841156e6d8c00a61f8050634344ee8b
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sun May 4 21:39:25 2025 +0200
Use replacements for HTML snippets instead of ELLIPSIS.
diff --git a/projects2.py b/projects2.py
index 4ab23d6..c3c80ef 100755
--- a/projects2.py
+++ b/projects2.py
@@ -463,12 +463,19 @@ class Projects2:
def upload_artifact_server(self, project, destination, md5_digest, user_id):
"""
>>> events = Events()
- >>> Projects2.create_null(
+ >>> projects2 = Projects2.create_null(
... args=["upload-artifact-server", "timeline", "1.0.0/file.exe", "a906449d5769fa7361d7ecc6aa3f6d28", "user_id"],
... stdinbuffer=b"123abc",
... events=events,
- ... ).run()
- >>> events # doctest: +ELLIPSIS
+ ... )
+ >>> projects2.run()
+ >>> events.replace(
+ ... projects2.generate_index_html(),
+ ... "<INDEX_HTML>"
+ ... ).replace(
+ ... projects2.generate_project_html("timeline"),
+ ... "<TIMELINE_HTML>"
+ ... )
WRITE_EVENT =>
path: '/opt/rlprojects/events'
project: 'timeline'
@@ -491,12 +498,12 @@ class Projects2:
LOG => 'Building /opt/rlprojects/web/index.html...'
WRITE =>
path: '/opt/rlprojects/web/index.html'
- ...
+ contents: '<INDEX_HTML>'
RUN => ['chown', 'scm:scm', '/opt/rlprojects/web/index.html']
LOG => 'Building /opt/rlprojects/web/timeline.html...'
WRITE =>
path: '/opt/rlprojects/web/timeline.html'
- ...
+ contents: '<TIMELINE_HTML>'
RUN => ['chown', 'scm:scm', '/opt/rlprojects/web/timeline.html']
>>> Projects2.create_null(
@@ -743,15 +750,25 @@ class Projects2:
... -----END EC PRIVATE KEY-----
... '''
>>> events = Events()
- >>> Projects2.create_null(
+ >>> projects2 = Projects2.create_null(
... args=["configure"],
... events=events,
... file_responses={
... CONFIG_INI_PATH: config,
... "/opt/rlprojects/web/git-project": None,
... }
- ... ).run()
- >>> events # doctest: +ELLIPSIS
+ ... )
+ >>> projects2.run()
+ >>> events.replace(
+ ... projects2.generate_index_html(),
+ ... "<INDEX_HTML>"
+ ... ).replace(
+ ... projects2.generate_project_html("git-project"),
+ ... "<GIT_PROJECT_HTML>"
+ ... ).replace(
+ ... projects2.generate_project_html("hg-project"),
+ ... "<HG_PROJECT_HTML>"
+ ... )
ENSURE_FOLDER =>
name: '/home/scm/.ssh'
user: 'scm'
@@ -891,23 +908,17 @@ class Projects2:
LOG => 'Building /opt/rlprojects/web/git-project.html...'
WRITE =>
path: '/opt/rlprojects/web/git-project.html'
- contents:
- <head>
- ...
+ contents: '<GIT_PROJECT_HTML>'
RUN => ['chown', 'scm:scm', '/opt/rlprojects/web/git-project.html']
LOG => 'Building /opt/rlprojects/web/hg-project.html...'
WRITE =>
path: '/opt/rlprojects/web/hg-project.html'
- contents:
- <head>
- ...
+ contents: '<HG_PROJECT_HTML>'
RUN => ['chown', 'scm:scm', '/opt/rlprojects/web/hg-project.html']
LOG => 'Building /opt/rlprojects/web/index.html...'
WRITE =>
path: '/opt/rlprojects/web/index.html'
- contents:
- <head>
- ...
+ contents: '<INDEX_HTML>'
RUN => ['chown', 'scm:scm', '/opt/rlprojects/web/index.html']
"""
self.setup_ssh()
@@ -1195,7 +1206,7 @@ server {{
SystemExit: Only pushes to main is allowed.
>>> events = Events()
- >>> Projects2.create_null(
+ >>> projects2 = Projects2.create_null(
... args=["pre-receive"],
... stdin="4b2a0166f479fd8b5e8cbc501360504965787edd 858b34e097c8563a60db02b19582136ad23057ae refs/heads/main\\n",
... events=events,
@@ -1220,8 +1231,15 @@ server {{
... process_responses={
... ("date",): {"stdout": "2025-04-18\\n"},
... },
- ... ).run()
- >>> events # doctest: +ELLIPSIS
+ ... )
+ >>> projects2.run()
+ >>> events.replace(
+ ... projects2.generate_index_html(),
+ ... "<INDEX_HTML>"
+ ... ).replace(
+ ... projects2.generate_project_html("test-project"),
+ ... "<TEST_PROJECT_HTML>"
+ ... )
LOG => 'Hello from projects2 pre-receive hook!'
RUN => ['mkdir', '/opt/rlprojects/tmp/tmp123/repo']
RUN => ['git', 'archive', '--format', 'tar', '-o', '/opt/rlprojects/tmp/tmp123/archive.tar', '858b34e097c8563a60db02b19582136ad23057ae']
@@ -1257,16 +1275,12 @@ server {{
LOG => 'Building /opt/rlprojects/web/index.html...'
WRITE =>
path: '/opt/rlprojects/web/index.html'
- contents:
- <head>
- ...
+ contents: '<INDEX_HTML>'
RUN => ['chown', 'scm:scm', '/opt/rlprojects/web/index.html']
LOG => 'Building /opt/rlprojects/web/test-project.html...'
WRITE =>
path: '/opt/rlprojects/web/test-project.html'
- contents:
- <head>
- ...
+ contents: '<TEST_PROJECT_HTML>'
RUN => ['chown', 'scm:scm', '/opt/rlprojects/web/test-project.html']
"""
self.logger.log("Hello from projects2 pre-receive hook!")
@@ -1289,7 +1303,7 @@ server {{
def pretxnchangegroup(self):
"""
>>> events = Events()
- >>> Projects2.create_null(
+ >>> projects2 = Projects2.create_null(
... args=["pretxnchangegroup"],
... stdin="4b2a0166f479fd8b5e8cbc501360504965787edd 858b34e097c8563a60db02b19582136ad23057ae refs/heads/main\\n",
... events=events,
@@ -1301,8 +1315,15 @@ server {{
... process_responses={
... ("date",): {"stdout": "2025-04-18\\n"},
... },
- ... ).run()
- >>> events # doctest: +ELLIPSIS
+ ... )
+ >>> projects2.run()
+ >>> events.replace(
+ ... projects2.generate_index_html(),
+ ... "<INDEX_HTML>"
+ ... ).replace(
+ ... projects2.generate_project_html("test-project"),
+ ... "<TEST_PROJECT_HTML>"
+ ... )
RUN => ['hg', 'clone', '-u', 'last-commit', '.', '/opt/rlprojects/tmp/tmp123']
WRITE_EVENT =>
path: '/opt/rlprojects/events'
@@ -1318,16 +1339,12 @@ server {{
LOG => 'Building /opt/rlprojects/web/index.html...'
WRITE =>
path: '/opt/rlprojects/web/index.html'
- contents:
- <head>
- ...
+ contents: '<INDEX_HTML>'
RUN => ['chown', 'scm:scm', '/opt/rlprojects/web/index.html']
LOG => 'Building /opt/rlprojects/web/test-project.html...'
WRITE =>
path: '/opt/rlprojects/web/test-project.html'
- contents:
- <head>
- ...
+ contents: '<TEST_PROJECT_HTML>'
RUN => ['chown', 'scm:scm', '/opt/rlprojects/web/test-project.html']
"""
with self.filesystem.temporary_directory(self.config.TMP_ROOT) as tmp:
commit 3a6448e792fe9cd3b5b1b159387f7e111656b68c
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sun May 4 21:30:58 2025 +0200
Get rid of one more ELLIPSIS.
diff --git a/projects2.py b/projects2.py
index 74c3d22..4ab23d6 100755
--- a/projects2.py
+++ b/projects2.py
@@ -364,13 +364,13 @@ class Projects2:
... args=["bootstrap"],
... events=events,
... ).run()
- >>> events.replace(repr(TEST_CONFIG_INI), "<config-ini>")
+ >>> events.replace(repr(TEST_CONFIG_INI), "<REPR_TEST_CONFIG_INI>")
RUN =>
command: ['ssh', 'root@projects.rickardlindberg.me', 'python3', '-', 'setup-bootstrap']
capture: False
stdin:
MYSELF = "CONFIG_INI_PATH = '/opt/rlprojects/config.ini'\\n"
- CONFIG_INI = <config-ini>
+ CONFIG_INI = <REPR_TEST_CONFIG_INI>
RUN => ['ssh', 'scm@projects.rickardlindberg.me', '/opt/rlprojects/rlprojects.py', 'sudo-configure']
"""
self.run_on_server(f"root@{server}", ["setup-bootstrap"])
@@ -386,13 +386,13 @@ class Projects2:
... args=["update"],
... events=events,
... ).run()
- >>> events # doctest: +ELLIPSIS
+ >>> events.replace(repr(TEST_CONFIG_INI), "<REPR_TEST_CONFIG_INI>")
RUN =>
command: ['ssh', 'scm@projects.rickardlindberg.me', 'python3', '-', 'install-self']
capture: False
stdin:
MYSELF = "CONFIG_INI_PATH = '/opt/rlprojects/config.ini'\\n"
- CONFIG_INI = '[Global]...'
+ CONFIG_INI = <REPR_TEST_CONFIG_INI>
RUN => ['ssh', 'scm@projects.rickardlindberg.me', '/opt/rlprojects/rlprojects.py', 'sudo-configure']
"""
self.run_on_server(f"{self.config.SSH_USER}@{server}", ["install-self"])
commit 8873781dda0d4eed5cf572dacd2006f5c108f30c
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sun May 4 21:29:12 2025 +0200
Get rid of ELLIPSIS is event output but supporting replacing things inside events.
diff --git a/projects2.py b/projects2.py
index d9b1e63..74c3d22 100755
--- a/projects2.py
+++ b/projects2.py
@@ -218,6 +218,29 @@ class WildcardCertificate:
print("KEY:")
print(self.key, end="")
+TEST_CONFIG_INI = """\
+[Global]
+InstanceName = rlprojects
+Domain = projects.rickardlindberg.me
+
+[User:admin]
+SshKeys = admin-ssh-key
+Projects = *
+
+[User:user-id]
+SshKeys = user-ssh-key
+Projects = timeline
+
+[WildcardCertificate]
+pem = -----BEGIN CERTIFICATE-----
+ -----END CERTIFICATE-----
+
+ -----BEGIN CERTIFICATE-----
+ -----END CERTIFICATE-----
+key = -----BEGIN EC PRIVATE KEY-----
+ -----END EC PRIVATE KEY-----
+"""
+
class Projects2:
@classmethod
@@ -248,28 +271,7 @@ class Projects2:
project_events={},
):
if CONFIG_INI_PATH not in file_responses:
- file_responses[CONFIG_INI_PATH] = """\
-[Global]
-InstanceName = rlprojects
-Domain = projects.rickardlindberg.me
-
-[User:admin]
-SshKeys = admin-ssh-key
-Projects = *
-
-[User:user-id]
-SshKeys = user-ssh-key
-Projects = timeline
-
-[WildcardCertificate]
-pem = -----BEGIN CERTIFICATE-----
- -----END CERTIFICATE-----
-
- -----BEGIN CERTIFICATE-----
- -----END CERTIFICATE-----
-key = -----BEGIN EC PRIVATE KEY-----
- -----END EC PRIVATE KEY-----
-"""
+ file_responses[CONFIG_INI_PATH] = TEST_CONFIG_INI
return cls(
args=Args.create_null(args=args),
process=Process.create_null(responses=process_responses).track(events),
@@ -362,13 +364,13 @@ key = -----BEGIN EC PRIVATE KEY-----
... args=["bootstrap"],
... events=events,
... ).run()
- >>> events # doctest: +ELLIPSIS
+ >>> events.replace(repr(TEST_CONFIG_INI), "<config-ini>")
RUN =>
command: ['ssh', 'root@projects.rickardlindberg.me', 'python3', '-', 'setup-bootstrap']
capture: False
stdin:
MYSELF = "CONFIG_INI_PATH = '/opt/rlprojects/config.ini'\\n"
- CONFIG_INI = '[Global]...'
+ CONFIG_INI = <config-ini>
RUN => ['ssh', 'scm@projects.rickardlindberg.me', '/opt/rlprojects/rlprojects.py', 'sudo-configure']
"""
self.run_on_server(f"root@{server}", ["setup-bootstrap"])
@@ -2304,8 +2306,24 @@ class Stdin:
class Events:
- def __init__(self):
- self.events = []
+ def __init__(self, events=None):
+ self.events = [] if events is None else events
+
+ def replace(self, thing, placeholder):
+ def replace(value):
+ if isinstance(value, str):
+ return value.replace(thing, placeholder)
+ elif isinstance(value, dict):
+ return {
+ key: replace(dict_value)
+ for key, dict_value in value.items()
+ }
+ else:
+ return value
+ return Events([
+ (name, replace(value))
+ for name, value in self.events
+ ])
def add(self, name, value):
self.events.append((name, value))
commit de91077b75955e802c3f0f29a0d84e330747db29
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sun May 4 21:22:46 2025 +0200
Get rid of an ELLIPSIS by string substitution.
diff --git a/projects2.py b/projects2.py
index adc9872..d9b1e63 100755
--- a/projects2.py
+++ b/projects2.py
@@ -1079,11 +1079,11 @@ server {{
... file_responses={
... CONFIG_INI_PATH: config,
... },
- ... ).generate_index_html()) # doctest: +ELLIPSIS
+ ... ).generate_index_html().replace(CSS, "<!--css-->"), end="")
<head>
<meta name="viewport" content="width=device-width,initial-scale=1">
<style type="text/css">
- ...
+ <!--css-->
</style>
<title>rlprojects</title>
</head>
@@ -1113,7 +1113,6 @@ server {{
</ul>
</div>
</div>
- <BLANKLINE>
"""
page = HtmlPage(title=self.config.TITLE)
page.add_column1(f"<h1>{self.config.TITLE}</h1>")
commit 4c1cb612f6b44263e315550f7902d0dc214b5a75
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sat May 3 11:46:55 2025 +0200
Remove unnecessary check.
diff --git a/projects2.py b/projects2.py
index 4fcaec0..adc9872 100755
--- a/projects2.py
+++ b/projects2.py
@@ -528,8 +528,6 @@ key = -----BEGIN EC PRIVATE KEY-----
)
with self.filesystem.temporary_directory(self.config.TMP_ROOT) as tmp:
source = join(tmp, basename(destination))
- if not self.filesystem.is_subpath(source, of=tmp):
- sys.exit(f"File {destination} is outside of artifact root.")
self.filesystem.write_binary(path=source, contents=contents)
self.install_artifact(
project=project,
commit dbfbd43d5e24ffff427636fc0dc68f62148db036
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sat May 3 11:39:11 2025 +0200
Ensure uploaded artifacts are not stored outside the root.
diff --git a/projects2.py b/projects2.py
index 0728c2e..4fcaec0 100755
--- a/projects2.py
+++ b/projects2.py
@@ -504,6 +504,15 @@ key = -----BEGIN EC PRIVATE KEY-----
Traceback (most recent call last):
...
SystemExit: Wrong checksum.
+
+ >>> Projects2.create_null(
+ ... args=["upload-artifact-server", "timeline", "../file.exe", "a906449d5769fa7361d7ecc6aa3f6d28", "user_id"],
+ ... stdinbuffer=b"123abc",
+ ... events=events,
+ ... ).run()
+ Traceback (most recent call last):
+ ...
+ SystemExit: File ../file.exe is outside of artifact root.
"""
contents, checksum = self.filesystem.md5()
if md5_digest == checksum:
@@ -519,6 +528,8 @@ key = -----BEGIN EC PRIVATE KEY-----
)
with self.filesystem.temporary_directory(self.config.TMP_ROOT) as tmp:
source = join(tmp, basename(destination))
+ if not self.filesystem.is_subpath(source, of=tmp):
+ sys.exit(f"File {destination} is outside of artifact root.")
self.filesystem.write_binary(path=source, contents=contents)
self.install_artifact(
project=project,
@@ -1437,24 +1448,12 @@ server {{
Traceback (most recent call last):
...
SystemExit: File ../report.txt is outside of artifact root.
-
- >>> join("/root", "/report")
- '/report'
-
- >>> join("/root", "../report")
- '/root/../report'
-
- >>> normpath("/root/")
- '/root'
-
- >>> normpath("/root/../report")
- '/report'
"""
- project_artifact_root = normpath(join(self.config.ARTIFACTS_ROOT, project))
- absolute_destination = normpath(join(project_artifact_root, destination))
+ project_artifact_root = join(self.config.ARTIFACTS_ROOT, project)
+ absolute_destination = join(project_artifact_root, destination)
if self.filesystem.exists(absolute_destination):
sys.exit(f"File {destination} already exists.")
- if not absolute_destination.startswith(f"{project_artifact_root}/"):
+ if not self.filesystem.is_subpath(absolute_destination, of=project_artifact_root):
sys.exit(f"File {destination} is outside of artifact root.")
self.fedora_system.ensure_folder(
name=dirname(absolute_destination),
@@ -1997,6 +1996,30 @@ class Filesystem:
self.os = os
self.sys = sys
+ def is_subpath(self, path, of):
+ """
+ >>> filesystem = Filesystem.create_null()
+ >>> filesystem.is_subpath("/foo/bar", of="/foo")
+ True
+ >>> filesystem.is_subpath("/foo/bar", of="/bar")
+ False
+
+ >>> join("/root", "/report")
+ '/report'
+
+ >>> join("/root", "../report")
+ '/root/../report'
+
+ >>> normpath("/root/")
+ '/root'
+
+ >>> normpath("/root/../report")
+ '/report'
+ """
+ path_parts = normpath(path).split("/")
+ of_parts = normpath(of).split("/")
+ return path_parts[:len(of_parts)] == of_parts
+
def get_modification_time(self, path):
"""
UNIX timestamps are always in UTC.
fatal: Invalid revision range 32fdc5358dfaa8a203526528a9d30c6317ecaf13..1517552e8b934a9d190514183b7111169aa8fbf1
commit 236966beea6b501ca40e632f334be5b0c6fba656
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sat May 3 11:05:15 2025 +0200
Render url to website if it exists.
diff --git a/projects2.py b/projects2.py
index 6d131e0..0728c2e 100755
--- a/projects2.py
+++ b/projects2.py
@@ -141,6 +141,10 @@ class Config:
def get_site_root(self, project):
return join(self.WEB_ROOT, project)
+ def get_site_url(self, project):
+ part = relpath(self.get_site_root(project), start=self.WEB_ROOT)
+ return f"https://{self.DOMAIN}/{part}"
+
def display_name(self, user_id):
return self.USERS.get(user_id, {}).get("display_name", user_id)
@@ -733,6 +737,7 @@ key = -----BEGIN EC PRIVATE KEY-----
... events=events,
... file_responses={
... CONFIG_INI_PATH: config,
+ ... "/opt/rlprojects/web/git-project": None,
... }
... ).run()
>>> events # doctest: +ELLIPSIS
@@ -1142,6 +1147,10 @@ server {{
page.add_column1("<p><a href=\"index.html\">Home</a></p>")
page.add_column2(f"<h2>Source code</h2>")
page.add_column2(f"<code>{self.clone_instructions(project)}</code>")
+ if self.filesystem.exists(self.config.get_site_root(project)):
+ url = html.escape(self.config.get_site_url(project))
+ page.add_column2(f"<h2>Website</h2>")
+ page.add_column2(f"<a href=\"{url}\">{url}</a>")
project_events = self.event_service.load(self.config.EVENTS_ROOT)
page.add_column3("<h2>Recent events</h2>")
for event in project_events.get_recent(project=project):
commit 44fda231be5fae23f9818960ca73ce80250198e7
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sat May 3 10:51:53 2025 +0200
Ensure site root exists before rsyncing files there.
diff --git a/projects2.py b/projects2.py
index 1c1b03d..6d131e0 100755
--- a/projects2.py
+++ b/projects2.py
@@ -138,6 +138,9 @@ class Config:
self.PROJECT_NAME_ENV_VAR = "PROJECT_NAME"
self.USER_ID_ENV_VAR = "USER_ID"
+ def get_site_root(self, project):
+ return join(self.WEB_ROOT, project)
+
def display_name(self, user_id):
return self.USERS.get(user_id, {}).get("display_name", user_id)
@@ -1219,6 +1222,10 @@ server {{
RUN => ['sudo', 'docker', 'build', '-f', 'Dockerfile.ci', '-q', '.']
LOG => 'Running Dockerfile.ci...'
RUN => ['sudo', 'timeout', '-k', '5s', '1m', 'docker', 'run', '--init', '--rm', '-v', '/opt/rlprojects/tmp/tmp123/repo:/repo:z', '-w', '/repo', '--user', ':', '']
+ ENSURE_FOLDER =>
+ name: '/opt/rlprojects/web/test-project'
+ user: 'scm'
+ group: 'scm'
RUN => ['rsync', '--archive', '--checksum', '--no-times', '--delete', '--verbose', '/opt/rlprojects/tmp/tmp123/repo/target/web/', '/opt/rlprojects/web/test-project']
ENSURE_FOLDER =>
name: '/opt/rlprojects/web/artifacts/test-project'
@@ -1343,12 +1350,17 @@ server {{
"--user", f"{user_id}:{group_id}",
image
], capture=False)
- site_root = f"{self.config.WEB_ROOT}/{project}"
+ site_root = self.config.get_site_root(project)
files_json = self.merge_artifacts([
self.filesystem.read_json(files)
for files in self.filesystem.ls(f"{repo_path}/Dockerfile*.ci.files")
])
if files_json["site"]:
+ self.fedora_system.ensure_folder(
+ name=site_root,
+ user=self.config.SSH_USER,
+ group=self.config.SSH_USER,
+ )
self.process.ensure([
"rsync",
"--archive",
fatal: Invalid revision range db166118e93630904722ab1b93f793bdd05b4e63..32fdc5358dfaa8a203526528a9d30c6317ecaf13
fatal: Invalid revision range 67cc4fcf07916d52c833885455622b3fee2ec2ec..db166118e93630904722ab1b93f793bdd05b4e63
commit 49735f07101d4372859c80d1398dbdb90caa436f
Author: Rickard Lindberg <rickard@rickardlindberg.me>
Date: Sat May 3 10:28:07 2025 +0200
Use checksum to determina if files have changed.
diff --git a/projects2.py b/projects2.py
index 15135bb..1c1b03d 100755
--- a/projects2.py
+++ b/projects2.py
@@ -1219,7 +1219,7 @@ server {{
RUN => ['sudo', 'docker', 'build', '-f', 'Dockerfile.ci', '-q', '.']
LOG => 'Running Dockerfile.ci...'
RUN => ['sudo', 'timeout', '-k', '5s', '1m', 'docker', 'run', '--init', '--rm', '-v', '/opt/rlprojects/tmp/tmp123/repo:/repo:z', '-w', '/repo', '--user', ':', '']
- RUN => ['rsync', '--archive', '--no-times', '--delete', '--verbose', '/opt/rlprojects/tmp/tmp123/repo/target/web/', '/opt/rlprojects/web/test-project']
+ RUN => ['rsync', '--archive', '--checksum', '--no-times', '--delete', '--verbose', '/opt/rlprojects/tmp/tmp123/repo/target/web/', '/opt/rlprojects/web/test-project']
ENSURE_FOLDER =>
name: '/opt/rlprojects/web/artifacts/test-project'
user: 'scm'
@@ -1352,6 +1352,7 @@ server {{
self.process.ensure([
"rsync",
"--archive",
+ "--checksum",
"--no-times",
"--delete",
"--verbose",