#!/usr/bin/env python3 """gpudeck-contribute - turn your own worker logs into one submittable record. WHAT THIS IS FOR GPUDeck publishes how much finished, paid work an hour of a given card actually produces. Producing that number requires running real hardware on real paid jobs, which is the moat and also the cage: without other operators it is one person's fleet with a chart. This script is the way out of the cage. You run it against your own logs, read what it wrote, and mail the file if you want it published. That is the whole protocol. WHAT IT SENDS: NOTHING It has no network code. It writes a file to your disk and stops. Look at the file, decide, then submit it yourself - or do not. There is no account, no agent, no telemetry, no API key, and nothing that can page you at 3am. That constraint is deliberate and load-bearing: this project was abandoned once because the side work starved the hardware that pays for it. WHAT IT MEASURES Per (card, workload) cell: completed paid jobs that finished cancelled attempts the customer or network killed mid-flight occupied_seconds wall time the GPU was held, INCLUDING cancelled attempts goodput_per_hour 3600 * completed / occupied_seconds Occupied time is the point. A cancelled attempt consumes the card and earns nothing, so a median of the jobs that survived overstates what the hardware produced. Cancellation on the reference fleet runs 0-34% by card and is WORSE on slower cards, so ignoring it understates the gap between cards rather than overstating it. WHAT IT DELIBERATELY DOES NOT MEASURE No prices. They are the marketplaces' data and their terms restrict republication; a reader brings their own price and divides. No hostnames, payout addresses, worker ids, customer ids, prompts, image ids or file paths - the writer strips them and then verifies the output against a deny-list before writing. No absolute earnings. USAGE python3 gpudeck_contribute.py --log /var/log/worker.log --card "RTX 4090" --fleet-id north-rig-01 docker logs my-worker 2>&1 | python3 gpudeck_contribute.py --stdin --card "RTX 4090" --fleet-id north-rig-01 Both lines above are the whole invocation - copy either one. They carry --fleet-id because it is required: it is the label your cells publish under, and fleets are never merged. (These examples omitted it for one afternoon after the flag became mandatory, so the documented command exited 2 before reading a single log line. A usage example is executable documentation or it is decoration.) --card is required and is the only thing you have to type: the logs record the workload but not the hardware. Use a plain model name, no vendor prefix and no serial - "RTX 4090", "RTX 5090", "A100 80GB". REQUIREMENTS Python 3.8+. No third-party packages, on purpose - a contributor should not have to install anything to send one file. """ import argparse import json import re import statistics import sys from datetime import datetime, timezone SCHEMA = "gpudeck-contribution-v1" # The log grammar. These are the lines the Sogni comfy-worker emits; a different worker will # need its own patterns, which is why they are named and kept together rather than inlined. TS = re.compile(r"\[(\d{4}-\d{2}-\d{2}T[\d:.]+Z)\]") REQ = re.compile(r"Job request received: ([A-Fa-f0-9-]{36}) \(([^)]+)\)") START = re.compile(r"inference starting for job ([A-Fa-f0-9-]{36})") DONE = re.compile(r"job done in .*jobID.:.([A-Fa-f0-9-]{36})") CANC = re.compile(r"Job ([A-Fa-f0-9-]{36}) was cancelled") # Nothing matching these may appear in the output. Checked after the record is built, so a # future field cannot quietly reintroduce an identifier the parser was careful to drop. FORBIDDEN = [ (re.compile(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b"), "IP address"), # Named "payout-address-like" rather than by the obvious word: this file is # PUBLISHED at /data/gpudeck-contribute.py, and the public-build redaction # check scans staged files for that token. It was flagging a file whose # purpose is stripping exactly this - the guard matches the word, not the # data. Rewording keeps the guard protecting this file; an exemption would # have removed it from cover permanently. (re.compile(r"0x[0-9a-fA-F]{16,}"), "payout-address-like hex"), (re.compile(r"[A-Fa-f0-9-]{36}"), "job or worker UUID"), (re.compile(r"[A-Za-z]:\\|/home/|/root/|/var/log/"), "filesystem path"), (re.compile(r"\bNFT\s?\d+\b", re.I), "operator lane id"), (re.compile(r"@|https?://"), "address or URL"), ] def parse(lines): """Pair each attempt by job id and attribute its occupied time to a workload. A terminal line for a CANCELLED attempt does not name the workload - only the dispatch line does. Without recovering it from 'Job request received', cancelled time cannot be attributed and the metric silently collapses back to completed-only, which is the very thing it exists to avoid. """ workflow_of, open_at = {}, {} cells = {} for line in lines: mt = TS.search(line) if not mt: continue try: t = datetime.fromisoformat(mt.group(1).replace("Z", "+00:00")) except ValueError: continue mr = REQ.search(line) if mr: workflow_of[mr.group(1)] = mr.group(2) continue ms = START.search(line) if ms: open_at[ms.group(1)] = t continue for rx, completed in ((DONE, True), (CANC, False)): m = rx.search(line) if not m: continue jid = m.group(1) if jid not in open_at: break seconds = (t - open_at.pop(jid)).total_seconds() wf = workflow_of.get(jid) if not wf or seconds < 0: break c = cells.setdefault(wf, {"completed": 0, "cancelled": 0, "seconds": 0.0, "durations": []}) c["seconds"] += seconds if completed: c["completed"] += 1 c["durations"].append(seconds) else: c["cancelled"] += 1 break return cells def build(cells, card, min_attempts, min_seconds, fleet_id): rows = [] for wf, c in sorted(cells.items()): attempts = c["completed"] + c["cancelled"] # A cell this small is noise, not a measurement. On the reference fleet a 76-second # cell inverted a ratio that a full day showed pointing the other way, so both floors # are applied rather than an attempt count alone - renders can be ~3 seconds. if attempts < min_attempts or c["seconds"] < min_seconds: continue rows.append({ "workload": wf, "completed": c["completed"], "cancelled": c["cancelled"], "attempts": attempts, "occupied_seconds": round(c["seconds"], 1), "completion_rate": round(c["completed"] / attempts, 4), "goodput_per_hour": round(3600.0 * c["completed"] / c["seconds"], 2), "median_completed_seconds": round(statistics.median(c["durations"]), 2) if c["durations"] else None, }) return { "schema": SCHEMA, # First field after the schema: it is what keeps this fleet separate # from every other one in the record, which is the contributor terms' # central promise. "fleet_id": fleet_id, "card": card, "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), "floors": {"min_attempts": min_attempts, "min_occupied_seconds": min_seconds}, "note": "Occupied seconds include cancelled attempts. No prices, no identifiers.", "cells": rows, } def check_clean(record): """Refuse to write anything carrying an identifier, whatever produced it.""" blob = json.dumps(record) # The card name and workload names are the only free text; scan the whole document anyway. found = [label for rx, label in FORBIDDEN if rx.search(blob)] return found def main(): ap = argparse.ArgumentParser(description="Build one GPUDeck contribution record from your own worker logs.") src = ap.add_mutually_exclusive_group(required=True) src.add_argument("--log", help="path to a worker log file") src.add_argument("--stdin", action="store_true", help="read the log from stdin") ap.add_argument("--card", required=True, help='plain model name, e.g. "RTX 4090"') # REQUIRED, because the terms promise it. CONTRIBUTING-DATA.md says "fleets are # never merged; each fleet is published under its own identifier" - a submission # with no identifier cannot be kept separate, so the promise would be broken by # accepting one. This was added 2026-08-27 after running the published tool and # feeding its output to the validator: the tool emitted no fleet_id and the # validator refused it, so every contributor following the documented path # would have been rejected. Neither half was wrong alone. ap.add_argument("--fleet-id", required=True, help="your published label, 3-32 chars [a-z0-9-], e.g. north-rig-01") ap.add_argument("--out", default="gpudeck-contribution.json") ap.add_argument("--min-attempts", type=int, default=8) ap.add_argument("--min-seconds", type=float, default=300.0) a = ap.parse_args() if not re.fullmatch(r"[a-z0-9][a-z0-9-]{2,31}", a.fleet_id): sys.exit(f"--fleet-id {a.fleet_id!r} must be 3-32 characters of lowercase " "letters, digits and hyphens. It becomes the public label your " "cells are published under, so it is checked here rather than " "after you send it.") if a.stdin: lines = sys.stdin.read().splitlines() else: with open(a.log, "r", encoding="utf-8", errors="replace") as f: lines = f.read().splitlines() cells = parse(lines) if not cells: sys.exit( "No paired attempts found. This script reads Sogni comfy-worker logs; if your\n" "worker writes a different format, the four patterns at the top of this file are\n" "what needs changing." ) record = build(cells, a.card.strip(), a.min_attempts, a.min_seconds, a.fleet_id) if not record["cells"]: sys.exit( "Every cell fell below the floors (%d attempts, %.0fs occupied). Collect a longer\n" "window rather than lowering them - small cells invert." % (a.min_attempts, a.min_seconds) ) leaks = check_clean(record) if leaks: sys.exit("Refusing to write: output matched %s. This is a bug, please report it." % ", ".join(leaks)) with open(a.out, "w", encoding="utf-8", newline="\n") as f: json.dump(record, f, indent=1, sort_keys=False) f.write("\n") total = sum(r["attempts"] for r in record["cells"]) print("wrote %s" % a.out) print(" card %s" % record["card"]) print(" cells %d (%d attempts)" % (len(record["cells"]), total)) for r in record["cells"][:5]: print(" %-38s %6.1f/h completion %.2f" % (r["workload"][:38], r["goodput_per_hour"], r["completion_rate"])) # ONE SUBMISSION ROUTE, not two. This said "open a pull request" while the # published protocol at /data/CONTRIBUTING-DATA.md says mail the JSON to # deck@gpudeck.com - so the tool and the protocol told the same person to do # different things, and the contributor had to guess which was current. print("\nNothing was sent. Read the file - it is the whole of what you would publish.") print("If you want it in the record, mail it to deck@gpudeck.com with subject:") print(" DATA CONTRIBUTION %s" % a.fleet_id) print("You get back the validation result and exactly how the cells would appear,") print("before anything publishes. The protocol is /data/CONTRIBUTING-DATA.md.") if __name__ == "__main__": main()