498 lines
49 KiB
Python
498 lines
49 KiB
Python
#!/usr/bin/env python3
|
|
"""Separate, guarded EJBCA Issuing migration. Run on the Docker host as root.
|
|
No root-CA resources are changed. Does not deploy/recreate the Issuing app.
|
|
Final output contains the requested new database password and Compose file.
|
|
"""
|
|
import collections
|
|
import datetime
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import time
|
|
|
|
MIGRATION_JAVA = 'import java.sql.*;\nimport java.util.*;\nimport java.util.stream.Collectors;\n\npublic final class IssuingMigration {\n private static final String H2_URL =\n "jdbc:h2:/source/restored;IFEXISTS=TRUE;ACCESS_MODE_DATA=r;NON_KEYWORDS=VALUE";\n private static final String PG_URL =\n "jdbc:postgresql://ejbca_issuing_migration_test_db:5432/ejbcadb";\n private static final int EXPECTED_TABLES = Integer.parseInt(System.getenv("ISSUING_EXPECTED_TABLES"));\n private static final long EXPECTED_ROWS = Long.parseLong(System.getenv("ISSUING_EXPECTED_ROWS"));\n private static boolean commitAttempted;\n private static boolean committed;\n\n private record Table(String schema, String name) {\n String sql() { return quote(schema) + "." + quote(name); }\n }\n private record Column(String name, int type, String typeName) {}\n private record Plan(Table source, Table target, List<Column> from, List<Column> to) {}\n private enum Kind { INT, LONG, BOOL, TEXT, BYTES }\n\n public static void main(String[] args) {\n try {\n require(args.length == 0 && EXPECTED_TABLES > 0 && EXPECTED_ROWS > 0, "Ungueltige Sollwerte/Argumente.");\n String password = System.getenv("TEST_PG_PASSWORD");\n require(password != null && !password.isEmpty(), "TEST_PG_PASSWORD fehlt.");\n Class.forName("org.h2.Driver");\n Class.forName("org.postgresql.Driver");\n try (Connection h2 = DriverManager.getConnection(H2_URL, "SA", System.getenv("ISSUING_H2_PASSWORD"))) {\n h2.setReadOnly(true);\n h2.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);\n h2.setAutoCommit(false);\n Properties props = new Properties();\n props.setProperty("user", "ejbca");\n props.setProperty("password", password);\n props.setProperty("connectTimeout", "15");\n props.setProperty("socketTimeout", "120");\n props.setProperty("ApplicationName", "IssuingMigration-test-only");\n props.setProperty("logServerErrorDetail", "false");\n try (Connection pg = DriverManager.getConnection(PG_URL, props)) {\n pg.setAutoCommit(false);\n try {\n migrate(h2, pg);\n commitAttempted = true;\n pg.commit(); // Einziger COMMIT im gesamten Programm.\n committed = true;\n System.out.println("COMMIT erfolgreich: " + EXPECTED_TABLES + " Tabellen, " + EXPECTED_ROWS + " Zeilen; Werte validiert.");\n } catch (Throwable failure) {\n try {\n pg.rollback();\n System.err.println("ROLLBACK angefordert und vom Server bestaetigt.");\n } catch (Throwable rollbackFailure) {\n System.err.println("ROLLBACK nicht bestaetigt; Testziel manuell pruefen.");\n }\n throw failure;\n }\n }\n }\n } catch (Throwable failure) {\n if (committed) {\n System.err.println("COMMIT war erfolgreich; Fehler beim Schliessen der Verbindung.");\n } else if (commitAttempted) {\n System.err.println("COMMIT-Ergebnis eventuell unbekannt. Testziel pruefen; kein automatischer Retry.");\n }\n if (failure instanceof GuardFailure) System.err.println(failure.getMessage());\n else if (failure instanceof SQLException sql)\n System.err.println("SQL-Abbruch, SQLState=" + sql.getSQLState() + ", Code=" + sql.getErrorCode());\n else System.err.println("Abbruch: " + failure.getClass().getSimpleName());\n System.exit(1);\n }\n }\n\n private static void migrate(Connection h2, Connection pg) throws SQLException {\n execute(pg, "SET LOCAL lock_timeout = \'15s\'");\n execute(pg, "SET LOCAL statement_timeout = \'120s\'");\n execute(pg, "SET LOCAL row_security = off");\n try (Statement st = pg.createStatement(); ResultSet rs = st.executeQuery(\n "SELECT current_database(), current_user, current_setting(\'session_replication_role\')")) {\n require(rs.next() && "ejbcadb".equals(rs.getString(1)) && "ejbca".equals(rs.getString(2))\n && "origin".equals(rs.getString(3)), "Unerwartete PostgreSQL-Sitzung.");\n }\n Map<String, Table> source = tables(h2, false);\n Map<String, Table> target = tables(pg, true);\n require(source.size() == EXPECTED_TABLES, "H2-Tabellenzahl weicht vom Inventar ab.");\n Set<String> expected = new TreeSet<>(source.keySet());\n require(!expected.contains("userdatasourcedata"), "Unerwartete H2-Tabelle userdatasourcedata.");\n expected.add("userdatasourcedata");\n require(target.keySet().equals(expected), "PostgreSQL-Tabellenmenge stimmt nicht mit der Quelle ueberein.");\n require(source.keySet().containsAll(List.of("admingroupdata", "accessrulesdata", "adminentitydata")),\n "Erwartete FK-Tabellen fehlen.");\n\n // Sperren werden vor der Leerheitspruefung genommen und bis COMMIT/ROLLBACK gehalten.\n execute(pg, "LOCK TABLE " + target.values().stream().map(Table::sql)\n .collect(Collectors.joining(", ")) + " IN ACCESS EXCLUSIVE MODE");\n require(tables(pg, true).equals(target), "Zielschema hat sich geaendert.");\n for (Table t : target.values()) require(count(pg, t) == 0, "Zieltabelle ist nicht leer: " + t.name());\n validateForeignKeys(pg, target);\n // Keine benutzerdefinierten Trigger/Regeln, die Daten oder externe Systeme veraendern.\n try (Statement st = pg.createStatement(); ResultSet rs = st.executeQuery(\n "SELECT count(*) FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid " +\n "JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=\'public\' AND NOT t.tgisinternal")) {\n rs.next(); require(rs.getLong(1) == 0, "Benutzerdefinierte Trigger sind nicht erlaubt.");\n }\n try (Statement st = pg.createStatement(); ResultSet rs = st.executeQuery(\n "SELECT count(*) FROM pg_rules WHERE schemaname=\'public\'")) {\n rs.next(); require(rs.getLong(1) == 0, "Benutzerdefinierte Regeln sind nicht erlaubt.");\n }\n List<String> order = new ArrayList<>(source.keySet());\n order.remove("admingroupdata"); order.add(0, "admingroupdata");\n List<Plan> plans = new ArrayList<>();\n Map<String, Long> counts = new TreeMap<>();\n long total = 0;\n for (String name : order) {\n Map<String, Column> from = columns(h2, source.get(name));\n Map<String, Column> to = columns(pg, target.get(name));\n require(from.keySet().equals(to.keySet()), "Spaltennamen weichen ab: " + name);\n for (String column : from.keySet())\n require(kind(from.get(column)) == kind(to.get(column)), "Datentypabweichung: " + name + "." + column);\n require(primaryKey(h2, source.get(name)).equals(primaryKey(pg, target.get(name))),\n "Primaerschluessel weichen ab: " + name);\n plans.add(new Plan(source.get(name), target.get(name), new ArrayList<>(from.values()), new ArrayList<>(to.values())));\n long n = count(h2, source.get(name)); counts.put(name, n); total += n;\n }\n require(total == EXPECTED_ROWS, "H2-Zeilenzahl weicht vom Inventar ab.");\n for (Plan plan : plans) {\n long inserted = copy(h2, pg, plan);\n require(inserted == counts.get(key(plan.source().name())), "Insert-Anzahl weicht ab.");\n }\n execute(pg, "SET CONSTRAINTS ALL IMMEDIATE");\n long verified = 0;\n for (Plan plan : plans) {\n long n = counts.get(key(plan.source().name()));\n require(count(h2, plan.source()) == n && count(pg, plan.target()) == n,\n "Rowcount-Abweichung: " + plan.target().name());\n require(rows(h2, plan.source(), plan.from()).equals(rows(pg, plan.target(), plan.to())),\n "Wertevergleich fehlgeschlagen: " + plan.target().name());\n verified += n;\n System.out.println(plan.target().name() + ": " + n + " Zeilen validiert");\n }\n require(verified == EXPECTED_ROWS, "Gesamtzahl nach Transfer falsch.");\n require(count(pg, target.get("userdatasourcedata")) == 0, "userdatasourcedata ist nicht leer.");\n require(tables(pg, true).equals(target), "Zielschema hat sich geaendert.");\n }\n\n private static Map<String, Table> tables(Connection c, boolean postgres) throws SQLException {\n String sql = postgres\n ? "SELECT n.nspname, c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace " +\n "WHERE c.relkind IN (\'r\',\'p\',\'f\',\'m\') AND n.nspname NOT IN (\'pg_catalog\',\'information_schema\') " +\n "AND n.nspname NOT LIKE \'pg_toast%\'"\n : "SELECT TABLE_SCHEMA, TABLE_NAME FROM INFORMATION_SCHEMA.TABLES " +\n "WHERE TABLE_TYPE=\'BASE TABLE\' AND TABLE_SCHEMA <> \'INFORMATION_SCHEMA\'";\n Map<String, Table> result = new TreeMap<>();\n try (Statement st = c.createStatement(); ResultSet rs = st.executeQuery(sql)) {\n while (rs.next()) {\n Table t = new Table(rs.getString(1), rs.getString(2));\n require(t.schema().equals(postgres ? "public" : "PUBLIC"), "Unerwartetes Benutzerschema: " + t.schema());\n require(result.put(key(t.name()), t) == null, "Mehrdeutiger Tabellenname.");\n }\n }\n return result;\n }\n private static Map<String, Column> columns(Connection c, Table t) throws SQLException {\n Map<String, Column> result = new TreeMap<>();\n try (Statement st = c.createStatement(); ResultSet rs = st.executeQuery("SELECT * FROM " + t.sql() + " WHERE 1=0")) {\n ResultSetMetaData md = rs.getMetaData();\n for (int i = 1; i <= md.getColumnCount(); i++) {\n Column col = new Column(md.getColumnName(i), md.getColumnType(i), md.getColumnTypeName(i));\n kind(col); // Nicht unterstuetzte Typen auch in leeren Tabellen ablehnen.\n require(result.put(key(col.name()), col) == null, "Mehrdeutiger Spaltenname.");\n }\n }\n return result;\n }\n private static Set<String> primaryKey(Connection c, Table t) throws SQLException {\n Set<String> result = new TreeSet<>();\n try (ResultSet rs = c.getMetaData().getPrimaryKeys(null, t.schema(), t.name())) {\n while (rs.next()) result.add(key(rs.getString("COLUMN_NAME")));\n }\n require(!result.isEmpty(), "Primaerschluessel fehlt: " + t.name());\n return result;\n }\n private static void validateForeignKeys(Connection c, Map<String, Table> tables) throws SQLException {\n Set<String> actual = new TreeSet<>(); int count = 0;\n for (Table t : tables.values()) {\n try (ResultSet rs = c.getMetaData().getImportedKeys(null, t.schema(), t.name())) {\n while (rs.next()) {\n require("public".equals(rs.getString("PKTABLE_SCHEM")) && rs.getShort("KEY_SEQ") == 1,\n "Unerwarteter Foreign Key.");\n actual.add(key(t.name()) + "." + key(rs.getString("FKCOLUMN_NAME")) + "->"\n + key(rs.getString("PKTABLE_NAME")) + "." + key(rs.getString("PKCOLUMN_NAME")));\n count++;\n }\n }\n }\n require(count == 2 && actual.equals(Set.of(\n "accessrulesdata.admingroupdata_accessrules->admingroupdata.pk",\n "adminentitydata.admingroupdata_adminentities->admingroupdata.pk")), "Foreign Keys weichen vom Vorlauf ab.");\n }\n private static Kind kind(Column col) {\n return switch (col.type()) {\n case Types.INTEGER -> Kind.INT;\n case Types.BIGINT -> Kind.LONG;\n case Types.BOOLEAN, Types.BIT -> Kind.BOOL;\n case Types.CHAR, Types.VARCHAR, Types.LONGVARCHAR, Types.NCHAR,\n Types.NVARCHAR, Types.LONGNVARCHAR, Types.CLOB, Types.NCLOB -> Kind.TEXT;\n case Types.BINARY, Types.VARBINARY, Types.LONGVARBINARY, Types.BLOB -> Kind.BYTES;\n default -> throw new GuardFailure("Nicht unterstuetzter Typ: " + col.typeName());\n };\n }\n private static Object read(ResultSet rs, int i, Column col) throws SQLException {\n Object value = switch (kind(col)) {\n case INT -> rs.getInt(i);\n case LONG -> rs.getLong(i);\n case BOOL -> rs.getBoolean(i);\n case TEXT -> rs.getString(i);\n case BYTES -> rs.getBytes(i);\n };\n return rs.wasNull() ? null : value;\n }\n private static void bind(PreparedStatement ps, int i, Object value, Column col) throws SQLException {\n if (value == null) { ps.setNull(i, col.type()); return; }\n switch (kind(col)) {\n case INT -> ps.setInt(i, (Integer) value);\n case LONG -> ps.setLong(i, (Long) value);\n case BOOL -> ps.setBoolean(i, (Boolean) value);\n case TEXT -> ps.setString(i, (String) value);\n case BYTES -> ps.setBytes(i, (byte[]) value);\n }\n }\n private static String names(List<Column> cols) {\n return cols.stream().map(c -> quote(c.name())).collect(Collectors.joining(", "));\n }\n private static long copy(Connection h2, Connection pg, Plan plan) throws SQLException {\n String select = "SELECT " + names(plan.from()) + " FROM " + plan.source().sql();\n String insert = "INSERT INTO " + plan.target().sql() + " (" + names(plan.to()) + ") VALUES ("\n + String.join(",", Collections.nCopies(plan.to().size(), "?")) + ")";\n long n = 0;\n try (Statement st = h2.createStatement(); ResultSet rs = st.executeQuery(select);\n PreparedStatement ps = pg.prepareStatement(insert)) {\n while (rs.next()) {\n require(++n <= EXPECTED_ROWS, "Unerwartet viele Quellzeilen.");\n for (int i = 0; i < plan.from().size(); i++)\n bind(ps, i + 1, read(rs, i + 1, plan.from().get(i)), plan.to().get(i));\n require(ps.executeUpdate() == 1, "Insert hat nicht genau eine Zeile erzeugt.");\n }\n }\n return n;\n }\n // Vollstaendiger Multimengenvergleich ohne Sortierungs-/Collation-Abhaengigkeit.\n // Base64 fuer byte[] ist verlustfrei; null bleibt von leerem Text/Binaerwert getrennt.\n private static Map<List<Object>, Integer> rows(Connection c, Table t, List<Column> cols) throws SQLException {\n Map<List<Object>, Integer> result = new HashMap<>(); long n = 0;\n try (Statement st = c.createStatement(); ResultSet rs = st.executeQuery("SELECT " + names(cols) + " FROM " + t.sql())) {\n while (rs.next()) {\n require(++n <= EXPECTED_ROWS, "Unerwartet viele Zeilen beim Wertevergleich.");\n List<Object> row = new ArrayList<>();\n for (int i = 0; i < cols.size(); i++) {\n Object value = read(rs, i + 1, cols.get(i));\n row.add(value instanceof byte[] bytes ? Base64.getEncoder().encodeToString(bytes) : value);\n }\n result.merge(row, 1, Integer::sum);\n }\n }\n return result;\n }\n private static long count(Connection c, Table t) throws SQLException {\n try (Statement st = c.createStatement(); ResultSet rs = st.executeQuery("SELECT COUNT(*) FROM " + t.sql())) {\n require(rs.next(), "COUNT lieferte kein Ergebnis."); return rs.getLong(1);\n }\n }\n private static void execute(Connection c, String sql) throws SQLException {\n try (Statement st = c.createStatement()) { st.execute(sql); }\n }\n private static String key(String s) { return s.toLowerCase(Locale.ROOT); }\n private static String quote(String s) { return "\\"" + s.replace("\\"", "\\"\\"") + "\\""; }\n private static void require(boolean condition, String message) {\n if (!condition) throw new GuardFailure(message);\n }\n private static final class GuardFailure extends RuntimeException {\n private static final long serialVersionUID = 1L;\n GuardFailure(String message) { super(message); }\n }\n}\n'
|
|
INVENTORY_JAVA = 'import java.sql.*;\nimport java.nio.file.*;\nimport java.nio.charset.StandardCharsets;\nimport java.security.MessageDigest;\nimport java.util.*;\nimport java.io.*;\npublic class Inventory {\n static String q(String s) { return "\\""+s.replace("\\"","\\"\\"")+"\\""; }\n public static void main(String[] args) {\n try { run(args); } catch(Exception e) {\n System.err.println("H2 verification failed: "+e.getClass().getSimpleName()); System.exit(1);\n }\n }\n static void run(String[] args) throws Exception {\n String pass=System.getenv("ISSUING_H2_PASSWORD");\n if(pass==null || pass.isEmpty()) throw new IllegalStateException();\n Class.forName("org.h2.Driver");\n boolean restore=args.length==1 && args[0].equals("restore");\n String url="jdbc:h2:/source/restored;NON_KEYWORDS=VALUE"+\n (restore ? "" : ";IFEXISTS=TRUE;ACCESS_MODE_DATA=r");\n try(Connection c=DriverManager.getConnection(url,"SA",pass)) {\n if(restore) {\n try(Reader r=Files.newBufferedReader(Path.of("/source/restore.sql"),StandardCharsets.UTF_8)) {\n org.h2.tools.RunScript.execute(c,r);\n }\n return;\n }\n c.setReadOnly(true); c.setAutoCommit(false);\n List<String> names=new ArrayList<>();\n try(Statement s=c.createStatement(); ResultSet r=s.executeQuery(\n "SELECT TABLE_SCHEMA,TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE=\'BASE TABLE\' AND TABLE_SCHEMA <> \'INFORMATION_SCHEMA\'")) {\n while(r.next()) {\n if(!r.getString(1).equals("PUBLIC")) throw new IllegalStateException();\n names.add(r.getString(2));\n }\n }\n Collections.sort(names);\n for(String name:names) {\n if(!name.matches("[A-Za-z0-9_]+")) throw new IllegalStateException();\n String t="PUBLIC."+q(name);\n try(Statement s=c.createStatement(); ResultSet r=s.executeQuery("SELECT COUNT(*) FROM "+t)) {\n r.next(); System.out.println("TABLE\\t"+name+"\\t"+r.getLong(1));\n }\n try(Statement s=c.createStatement();ResultSet r=s.executeQuery("SELECT * FROM "+t)) {\n ResultSetMetaData m=r.getMetaData();\n List<Integer> bin=new ArrayList<>();\n for(int i=1;i<=m.getColumnCount();i++) {\n int type=m.getColumnType(i);\n if(type==Types.BINARY || type==Types.VARBINARY || type==Types.LONGVARBINARY || type==Types.BLOB) {\n bin.add(i); System.out.println("COLUMN\\t"+name+"\\t"+(i-1)+"\\t"+m.getColumnName(i));\n }\n }\n while(r.next()) for(int i:bin) {\n byte[] b=r.getBytes(i);\n if(b!=null) System.out.println("BINARY\\t"+name+"\\t"+(i-1)+"\\t"+b.length+"\\t"+\n HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(b)));\n }\n }\n }\n c.rollback();\n }\n }\n}\n'
|
|
|
|
APP = 'ejbca_issuing'
|
|
ORIGINAL_VOLUME = 'ejbca_issuing_data'
|
|
TEST_DB = 'ejbca_issuing_migration_test_db'
|
|
TEST_APP = 'ejbca_issuing_migration_test_app'
|
|
FINAL_DB = 'ejbca_issuing_db'
|
|
TEST_NETWORK = 'ejbca-issuing-migration-test-net'
|
|
FINAL_NETWORK = 'ejbca-issuing-db-net'
|
|
TEST_DB_VOLUME = 'ejbca_issuing_migration_test_db_data'
|
|
FINAL_DB_VOLUME = 'ejbca_issuing_postgres_data'
|
|
TEST_APP_VOLUME = 'ejbca_issuing_migration_test_app_data'
|
|
FINAL_APP_VOLUME = 'ejbca_issuing_postgres_app_data'
|
|
EXPECTED_IMAGE = 'sha256:e63fb1bd5834154ea5a2f133a2d82b7e3fa5c0246924c7c3bed2d44b0c2d8a5a'
|
|
EJBCA_IMAGE = 'keyfactor/ejbca-ce@sha256:183b86af44b9b13e7cc8912c868f635aeb8dba6bf056ccbd4683b17626964d0a'
|
|
JAVA_IMAGE = 'eclipse-temurin:17-jdk'
|
|
PG_IMAGE = 'postgres:17'
|
|
BASE = Path('/root/ejbca-issuing-migration')
|
|
WORK = None
|
|
|
|
class MigrationError(Exception): pass
|
|
|
|
def check(ok, message):
|
|
if not ok: raise MigrationError(message)
|
|
|
|
def log(message):
|
|
print(message, flush=True)
|
|
if WORK:
|
|
with (WORK/'progress.log').open('a') as f: f.write(message+'\n')
|
|
|
|
def run(argv, *, data=None, env=None, timeout=600):
|
|
p = subprocess.run(argv, input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
env=env, timeout=timeout)
|
|
if p.returncode:
|
|
if WORK:
|
|
with (WORK/'private-errors.log').open('ab') as f:
|
|
f.write(p.stdout+b'\n'+p.stderr+b'\n')
|
|
raise MigrationError('Befehl fehlgeschlagen; Details nur lokal in private-errors.log. Keine Geheimnisse daraus posten.')
|
|
return p.stdout
|
|
|
|
def docker(*args, **kw): return run(['docker', *map(str,args)], **kw)
|
|
def inspect(name): return json.loads(docker('inspect',name))[0]
|
|
def envmap(config): return dict(x.split('=',1) for x in config.get('Env',[]) if '=' in x)
|
|
def sql(db, query, name='ejbcadb'):
|
|
return docker('exec','-i',db,'psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','ejbca','-d',name,
|
|
data=query.encode()).decode().strip()
|
|
def qi(s): return '"'+s.replace('"','""')+'"'
|
|
def ql(s): return "'"+s.replace("'","''")+"'"
|
|
def digest(path):
|
|
h=hashlib.sha256()
|
|
with path.open('rb') as f:
|
|
for b in iter(lambda:f.read(1024*1024), b''): h.update(b)
|
|
return h.hexdigest()
|
|
def absent(kind, names):
|
|
for n in names:
|
|
p=subprocess.run(['docker',kind,'inspect',n],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
check(p.returncode != 0, 'Bereits vorhanden: '+n+'. Keine automatische Wiederholung oder Loeschung.')
|
|
def mount(src,dst,readonly=True,kind='bind'):
|
|
return ['--mount',f'type={kind},src={src},dst={dst}'+(',readonly' if readonly else '')]
|
|
|
|
def java(classname, args=(), readonly=True, network='none', extra=None):
|
|
cmd=['run','--rm','--network',network]
|
|
cmd+=mount(WORK/'restored','/source',readonly)
|
|
cmd+=mount(WORK/'classes','/classes')+mount(WORK/'drivers','/drivers')
|
|
cmd+=['-e','ISSUING_H2_PASSWORD']
|
|
env=os.environ.copy(); env['ISSUING_H2_PASSWORD']=H2_PASSWORD
|
|
for k,v in (extra or {}).items(): cmd+=['-e',k]; env[k]=str(v)
|
|
cmd += [JAVA_IMAGE,'java','-cp','/classes:/drivers/*',classname,*args]
|
|
return docker(*cmd, env=env, timeout=3600).decode()
|
|
|
|
# A deliberately small SQL lexer: the Recover output must match its supported
|
|
# syntax. Strings (including doubled apostrophes) and quoted identifiers are
|
|
# preserved byte for byte. Unsupported constructs cause an abort.
|
|
def split_sql(text, delimiter=';'):
|
|
items=[]; start=0; quote=None; depth=0; i=0; comment=False
|
|
while i<len(text):
|
|
c=text[i]
|
|
if comment:
|
|
if c=='\n': comment=False
|
|
elif quote:
|
|
if c==quote:
|
|
if i+1<len(text) and text[i+1]==quote: i+=1
|
|
else: quote=None
|
|
elif c in "'\"": quote=c
|
|
elif text[i:i+2]=='--': comment=True; i+=1
|
|
elif c=='(': depth+=1
|
|
elif c==')': depth-=1
|
|
elif c==delimiter and (delimiter==';' or depth==0):
|
|
items.append(text[start:i].strip()); start=i+1
|
|
i+=1
|
|
check(quote is None and depth==0, 'Nicht unterstuetztes/unvollstaendiges SQL im Recover-Dump.')
|
|
if text[start:].strip(): items.append(text[start:].strip())
|
|
return items
|
|
|
|
def patch_recover(raw):
|
|
text=raw.decode('utf-8',errors='strict')
|
|
check(not re.search(r'^\s*//\s*error:',text,re.M|re.I), 'Recover meldet einen Fehler; Abbruch.')
|
|
mapping=dict(re.findall(r'INSERT INTO "PUBLIC"\."([A-Z0-9_]+)" SELECT \* FROM (O_\d+);',text))
|
|
reverse={v:k for k,v in mapping.items()}
|
|
check(len(reverse)==len(mapping) and mapping, 'Recover-Tabellenzuordnung nicht eindeutig.')
|
|
# Recover writes each staging INSERT on a single physical line; reject
|
|
# deviations rather than interpreting or modifying arbitrary SQL.
|
|
inserts=[]; binary_columns=set(); expected=collections.Counter()
|
|
for line in text.splitlines():
|
|
if not line.startswith('INSERT INTO O_'): continue
|
|
m=re.fullmatch(r'INSERT INTO (O_\d+) VALUES\((.*)\);',line)
|
|
check(m is not None, 'Mehrzeiliges/unerwartetes Recover-INSERT. Manuelle Pruefung erforderlich.')
|
|
helper,body=m.groups(); check(helper in reverse,'Unbekannte Recover-Hilfstabelle.')
|
|
vals=split_sql(body,','); inserts.append((helper,vals))
|
|
for col,v in enumerate(vals):
|
|
if re.match(r"(?i)^X'",v):
|
|
check(re.fullmatch(r"(?i)X'[0-9a-f]*'",v) is not None,'Unerwartetes Binaerliteral.')
|
|
b=bytes.fromhex(v[2:-1]); binary_columns.add((helper,col))
|
|
expected[(reverse[helper],col,len(b),hashlib.sha256(b).hexdigest())]+=1
|
|
check(inserts and binary_columns,'Keine erwarteten Daten/Binaerwerte im Recover-Dump.')
|
|
for helper,vals in inserts:
|
|
for h,col in binary_columns:
|
|
if h==helper:
|
|
check(col<len(vals) and (vals[col].upper()=='NULL' or
|
|
re.fullmatch(r"(?i)X'[0-9a-f]*'",vals[col])), 'Gemischte Typen in einer Binaerspalte.')
|
|
for helper in sorted({h for h,c in binary_columns}):
|
|
pat=rf'^CREATE TABLE {helper}\(([^\n]+)\);$'
|
|
matches=list(re.finditer(pat,text,re.M)); check(len(matches)==1,'Hilfstabellendefinition nicht eindeutig.')
|
|
old=matches[0].group(0); columns=split_sql(matches[0].group(1),',')
|
|
for h,col in sorted(binary_columns):
|
|
if h==helper:
|
|
check(columns[col]==f'C{col} VARCHAR','Unerwarteter Hilfsspalten-Typ.')
|
|
columns[col]=f'C{col} VARBINARY'
|
|
text=text.replace(old,f"CREATE TABLE {helper}({', '.join(columns)});",1)
|
|
text,n=re.subn(r'^CREATE USER IF NOT EXISTS "SA" .* ADMIN;$',
|
|
'CREATE USER IF NOT EXISTS "SA" PASSWORD '+ql(H2_PASSWORD)+' ADMIN;',text,flags=re.M)
|
|
check(n==1,'SA-Benutzerdefinition nicht eindeutig.')
|
|
return text,expected,collections.Counter(reverse[h].lower() for h,vals in inserts)
|
|
|
|
def read_inventory():
|
|
output=java('Inventory'); (WORK/'source-inventory.tsv').write_text(output)
|
|
counts={}; binaries=collections.Counter()
|
|
for line in output.splitlines():
|
|
x=line.split('\t')
|
|
if x[0]=='TABLE': counts[x[1].lower()]=int(x[2])
|
|
elif x[0]=='BINARY': binaries[(x[1],int(x[2]),int(x[3]),x[4])]+=1
|
|
elif x[0]!='COLUMN': raise MigrationError('Unerwartete Inventarausgabe.')
|
|
check(counts and sum(counts.values())>0,'Leere Quelle.')
|
|
return counts,binaries
|
|
|
|
def extract_image_files():
|
|
wanted={
|
|
'schema.sql':'opt/keyfactor/ejbca/doc/sql-scripts/create-tables-ejbca-postgres.sql',
|
|
'indexes-original.sql':'opt/keyfactor/ejbca/doc/sql-scripts/create-index-ejbca.sql',
|
|
'drivers/postgresql.jar':'opt/keyfactor/dbopt/postgres/jdbc-driver.jar'}
|
|
found={}; h2=[]
|
|
p=subprocess.Popen(['docker','export',APP],stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
|
try:
|
|
with tarfile.open(fileobj=p.stdout,mode='r|') as archive:
|
|
for entry in archive:
|
|
name=entry.name.removeprefix('./')
|
|
if not entry.isfile(): continue
|
|
for out,path in wanted.items():
|
|
if name==path:
|
|
(WORK/out).write_bytes(archive.extractfile(entry).read()); found[out]=True
|
|
if re.search(r'/(h2-[0-9][^/]*\.jar|dbopt/h2/jdbc-driver\.jar)$',name):
|
|
h2.append((name,archive.extractfile(entry).read()))
|
|
check(p.wait()==0,'Containerexport fehlgeschlagen.')
|
|
finally:
|
|
if p.poll() is None: p.kill(); p.wait()
|
|
check(len(found)==3 and h2,'Schema oder JDBC-Treiber fehlt im getesteten Image.')
|
|
hashes={hashlib.sha256(b).hexdigest() for n,b in h2}
|
|
check(len(hashes)==1,'Mehrere unterschiedliche H2-Treiber im Image.')
|
|
(WORK/'drivers/h2.jar').write_bytes(h2[0][1])
|
|
check(hashlib.sha256(h2[0][1]).hexdigest()=='b9d8f19358ada82a4f6eb5b174c6cfe320a375b5a9cb5a4fe456d623e6e55497',
|
|
'H2-Treiber entspricht nicht dem getesteten H2 2.2.224.')
|
|
|
|
|
|
def prepare_indexes():
|
|
src=(WORK/'indexes-original.sql').read_text()
|
|
active=[l.strip() for l in src.splitlines() if l.strip() and not l.lstrip().startswith('--')]
|
|
indexes=[]; deletes=[]
|
|
for line in active:
|
|
if line.startswith('CREATE '):
|
|
check(re.fullmatch(r'CREATE (?:UNIQUE )?INDEX \w+ ON \w+\s*\(.*\);',line), 'Unbekannte Indexsyntax.')
|
|
indexes.append(line)
|
|
elif line.startswith('DELETE '): deletes.append(line)
|
|
else: raise MigrationError('Unbekannter aktiver Befehl im Indexskript.')
|
|
fps=('caba75f68c833c3c2d33f3f5052b7d5a76e80383','05a219d835622653192c30eeeee8f01f918b30fb')
|
|
expected={f"DELETE FROM {t} WHERE fingerprint='{f}';" for t in ('CertificateData','Base64CertData') for f in fps}
|
|
check(set(deletes)==expected and len(deletes)==4 and len(indexes)==44,'Indexskript weicht vom getesteten Image ab.')
|
|
old='CREATE INDEX certificatedata_idx_san ON CertificateData (subjectAltName(750));'
|
|
check(indexes.count(old)==1,'SAN-Index nicht eindeutig.')
|
|
indexes[indexes.index(old)]='CREATE INDEX certificatedata_idx_san ON CertificateData (LEFT(subjectaltname,750));'
|
|
guard="""SET LOCAL lock_timeout='15s';
|
|
SET LOCAL statement_timeout='120s';
|
|
LOCK TABLE public.certificatedata, public.base64certdata IN SHARE ROW EXCLUSIVE MODE;
|
|
DO $guard$ BEGIN
|
|
IF EXISTS (SELECT 1 FROM public.certificatedata WHERE fingerprint IN ('caba75f68c833c3c2d33f3f5052b7d5a76e80383','05a219d835622653192c30eeeee8f01f918b30fb')
|
|
UNION ALL SELECT 1 FROM public.base64certdata WHERE fingerprint IN ('caba75f68c833c3c2d33f3f5052b7d5a76e80383','05a219d835622653192c30eeeee8f01f918b30fb')) THEN
|
|
RAISE EXCEPTION 'Test-Fingerprints vorhanden; keine automatische Loeschung'; END IF; END $guard$;
|
|
"""
|
|
(WORK/'indexes.sql').write_text(guard+'\n'.join(indexes)+'\n')
|
|
|
|
|
|
def create_pg(name,volume,network,password):
|
|
docker('volume','create',volume)
|
|
docker('run','-d','--name',name,'--restart','unless-stopped' if name==FINAL_DB else 'no',
|
|
'--network',network,'-e','POSTGRES_DB=ejbcadb','-e','POSTGRES_USER=ejbca','-e','POSTGRES_PASSWORD',
|
|
*mount(volume,'/var/lib/postgresql/data',False,'volume'),PG_PIN,
|
|
env={**os.environ,'POSTGRES_PASSWORD':password})
|
|
for _ in range(60):
|
|
p=subprocess.run(['docker','exec',name,'pg_isready','-h','127.0.0.1','-U','ejbca','-d','ejbcadb'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
if p.returncode==0: return
|
|
time.sleep(2)
|
|
raise MigrationError('PostgreSQL nicht bereit: '+name)
|
|
|
|
def load_sql(db,path):
|
|
docker('exec','-i',db,'psql','-X','-v','ON_ERROR_STOP=1','--single-transaction','-U','ejbca','-d','ejbcadb','-f','-',data=path.read_bytes())
|
|
|
|
def snapshot(db):
|
|
tables=json.loads(sql(db,"SELECT coalesce(json_agg(tablename ORDER BY tablename),'[]') FROM pg_tables WHERE schemaname='public';"))
|
|
result={}
|
|
for table in tables:
|
|
# Deterministic complete row representation, ordered as bytes by C collation.
|
|
q="SET bytea_output='hex'; SELECT count(*)::text||'|'||md5(coalesce(string_agg(v,E'\\n' ORDER BY v COLLATE \"C\"),'')) FROM (SELECT row_to_json(t)::text AS v FROM public."+qi(table)+" t) s;"
|
|
n,h=sql(db,q).split('|'); result[table]=(int(n),h)
|
|
idx=json.loads(sql(db,"SELECT json_build_object('count',count(*),'bad',count(*) FILTER(WHERE NOT i.indisvalid OR NOT i.indisready)) FROM pg_index i JOIN pg_class t ON t.oid=i.indrelid JOIN pg_namespace n ON n.oid=t.relnamespace WHERE n.nspname='public' AND NOT i.indisprimary;"))
|
|
check(idx=={'count':44,'bad':0},'Nicht genau 44 gueltige Zusatzindizes.')
|
|
return result
|
|
|
|
def migrate(counts):
|
|
output=java('IssuingMigration',network=TEST_NETWORK,extra={
|
|
'TEST_PG_PASSWORD':TEST_PASSWORD,'ISSUING_EXPECTED_TABLES':len(counts),
|
|
'ISSUING_EXPECTED_ROWS':sum(counts.values())})
|
|
(WORK/'migration.log').write_text(output)
|
|
check('COMMIT erfolgreich:' in output,'Kein bestaetigter Migrations-COMMIT.')
|
|
log('JDBC-Transfer und vollstaendiger Wertevergleich erfolgreich.')
|
|
|
|
def copy_volume(target):
|
|
check(inspect(APP)['State']['Status']=='exited','Original-Issuing wurde zwischenzeitlich gestartet.')
|
|
docker('volume','create',target)
|
|
docker('run','--rm','--network','none','--user','0:0','--entrypoint','/bin/sh',
|
|
*mount(ORIGINAL_VOLUME,'/source',True,'volume'),*mount(target,'/target',False,'volume'),
|
|
JAVA_IMAGE,'-c','set -e; cp -a /source/. /target/')
|
|
|
|
def app_cli(*args):
|
|
return docker('exec',TEST_APP,'/opt/keyfactor/ejbca/bin/ejbca.sh',*args,timeout=180).decode()
|
|
|
|
def test_application(original_env,counts):
|
|
copy_volume(TEST_APP_VOLUME)
|
|
keep=('PASSWORD_ENCRYPTION_KEY','EJBCA_CLI_DEFAULT_PASSWORD','EJBCA_CLI_DEFAULTPASSWORD',
|
|
'EJBCA_CLI_DEFAULT_USERNAME','TLS_SETUP_ENABLED','ADMINWEB_ACCESS','CA_KEYSTOREPASS')
|
|
values={k:original_env[k] for k in keep if k in original_env}
|
|
values.update(DATABASE_JDBC_URL=f'jdbc:postgresql://{TEST_DB}:5432/ejbcadb',DATABASE_USER='ejbca',
|
|
DATABASE_PASSWORD=TEST_PASSWORD,HTTPSERVER_HOSTNAME='issuing-migration-test')
|
|
args=['run','-d','--name',TEST_APP,'--label','issuing-migration.run='+str(WORK),'--restart','no','--network',TEST_NETWORK]
|
|
env=os.environ.copy()
|
|
for k,v in values.items(): args+=['-e',k]; env[k]=v
|
|
args+=mount(TEST_APP_VOLUME,'/mnt/persistent',False,'volume')+[EJBCA_IMAGE]
|
|
docker(*args,env=env)
|
|
ready=False
|
|
for attempt in range(120):
|
|
time.sleep(5)
|
|
if attempt%6==0: log('Warte auf EJBCA-Teststart ...')
|
|
p=subprocess.run(['docker','exec',TEST_APP,'curl','-fsS','--max-time','3',
|
|
'http://127.0.0.1:8090/health/ready'],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL)
|
|
if p.returncode==0:
|
|
try: ready=json.loads(p.stdout).get('status')=='UP'
|
|
except ValueError: pass
|
|
if ready: break
|
|
check(inspect(TEST_APP)['State']['Running'],'Testanwendung hat sich beendet.')
|
|
# docker logs may emit server output on stderr: retrieve both streams locally.
|
|
p=subprocess.run(['docker','logs',TEST_APP],stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
|
|
logs=p.stdout.decode(errors='replace'); (WORK/'test-start.private.log').write_text(logs)
|
|
check(ready,'EJBCA wurde nicht bereit; privates Startlog lokal pruefen.')
|
|
check(not re.search(r'\bERROR\b',logs),'ERROR im Teststart; keine automatische Freigabe.')
|
|
check('isUniqueCertificateSerialNumberIndex to: true' in logs,'Zertifikatsindex nicht erkannt.')
|
|
check('Initialized CA:' in logs,'Keine lokale CA initialisiert.')
|
|
(WORK/'test-ca-list.log').write_text(app_cli('ca','listcas'))
|
|
listing=app_cli('cryptotoken','list'); (WORK/'test-tokens.log').write_text(listing)
|
|
tokens=re.findall(r'"([^"\r\n]+)" \((-?\d+)\)\s+SoftCryptoToken, active,',listing)
|
|
check(len(tokens)==counts.get('cryptotokendata',0)>0,'Nicht alle Tokens sind aktive Software-Tokens.')
|
|
tested=0; post_checks=[]
|
|
with (WORK/'key-tests.log').open('w') as out:
|
|
for name,tid in tokens:
|
|
listing=app_cli('cryptotoken','listkeys','--token',name)
|
|
aliases=re.findall(r'\(main\)\s+(\S+)\s+(?:RSA|ECDSA|EC|Ed25519|Ed448|EdDSA|DSA)\s+',listing)
|
|
check(aliases,'Keine unterstuetzten Schluessel-Aliase gefunden.')
|
|
for alias in aliases:
|
|
result=app_cli('cryptotoken','testkey','--token',name,'--alias',alias)
|
|
check('Key pair tested successfully.' in result,'Schluesseltest fehlgeschlagen.')
|
|
out.write(name+'/'+alias+': OK\n'); tested+=1
|
|
import shlex
|
|
post_checks.append('docker exec ejbca_issuing /opt/keyfactor/ejbca/bin/ejbca.sh cryptotoken testkey --token '+shlex.quote(name)+' --alias '+shlex.quote(alias))
|
|
(WORK/'verify-after-deploy.sh').write_text('#!/bin/sh\nset -eu\ndocker exec ejbca_issuing curl -fsS --max-time 10 http://127.0.0.1:8090/health/ready\n'+ '\n'.join(post_checks)+'\n')
|
|
docker('stop','--time','30',TEST_APP)
|
|
check(inspect(TEST_APP)['State']['Status']=='exited','Testanwendung nicht gestoppt.')
|
|
log(f'EJBCA UP, alle {len(tokens)} Tokens und {tested} Schluessel erfolgreich getestet.')
|
|
|
|
|
|
def yaml_text(value,indent=0):
|
|
pad=' '*indent
|
|
if isinstance(value,dict):
|
|
if not value: return pad+'{}\n'
|
|
out=''
|
|
for k,v in value.items():
|
|
key=json.dumps(str(k),ensure_ascii=False)
|
|
if isinstance(v,(dict,list)) and v:
|
|
out+=pad+key+':\n'+yaml_text(v,indent+2)
|
|
else: out+=pad+key+': '+json.dumps(v,ensure_ascii=False)+'\n'
|
|
return out
|
|
if isinstance(value,list):
|
|
return ''.join(pad+'- '+json.dumps(v,ensure_ascii=False)+'\n' for v in value)
|
|
raise MigrationError('Unbekannter YAML-Typ.')
|
|
|
|
|
|
def write_compose(original,image_env):
|
|
c=original['Config']; hc=original['HostConfig']; env=envmap(c)
|
|
overrides={k:v for k,v in env.items() if image_env.get(k)!=v}
|
|
overrides.update(DATABASE_JDBC_URL=f'jdbc:postgresql://{FINAL_DB}:5432/ejbcadb',
|
|
DATABASE_USER='ejbca',DATABASE_PASSWORD=FINAL_PASSWORD)
|
|
for k in ('PASSWORD_ENCRYPTION_KEY','EJBCA_CLI_DEFAULTPASSWORD','EJBCA_CLI_DEFAULT_PASSWORD','EJBCA_CLI_DEFAULT_USERNAME'):
|
|
if k in env: overrides[k]=env[k]
|
|
# Compose performs interpolation in values; escape literal dollar signs.
|
|
overrides={k:v.replace('$','$$') for k,v in overrides.items()}
|
|
networks={}
|
|
for name,n in original['NetworkSettings']['Networks'].items():
|
|
entry={}; ipam=n.get('IPAMConfig') or {}
|
|
if ipam.get('IPv4Address'): entry['ipv4_address']=ipam['IPv4Address']
|
|
if ipam.get('IPv6Address'): entry['ipv6_address']=ipam['IPv6Address']
|
|
networks[name]=entry
|
|
networks[FINAL_NETWORK]={}
|
|
service={'image':EJBCA_IMAGE,'container_name':APP,'hostname':c['Hostname'],
|
|
'restart':hc['RestartPolicy'].get('Name') or 'no','environment':overrides,
|
|
'volumes':[FINAL_APP_VOLUME+':/mnt/persistent'],'networks':networks}
|
|
ports=[]
|
|
for port,bindings in (hc.get('PortBindings') or {}).items():
|
|
for b in bindings or []:
|
|
ip=b.get('HostIp',''); host=b['HostPort']; ports.append((ip+':' if ip else '')+host+':'+port)
|
|
if ports: service['ports']=ports
|
|
doc={'services':{APP:service},'volumes':{FINAL_APP_VOLUME:{'external':True}},
|
|
'networks':{n:{'external':True} for n in networks}}
|
|
(WORK/'docker-compose.issuing.yml').write_text(yaml_text(doc))
|
|
dbdoc={'services':{FINAL_DB:{'image':PG_PIN,'container_name':FINAL_DB,'restart':'unless-stopped',
|
|
'environment':{'POSTGRES_DB':'ejbcadb','POSTGRES_USER':'ejbca','POSTGRES_PASSWORD':FINAL_PASSWORD},
|
|
'volumes':[FINAL_DB_VOLUME+':/var/lib/postgresql/data'],'networks':{FINAL_NETWORK:{}}}},
|
|
'volumes':{FINAL_DB_VOLUME:{'external':True}},'networks':{FINAL_NETWORK:{'external':True}}}
|
|
(WORK/'docker-compose.database-reference.yml').write_text(yaml_text(dbdoc))
|
|
# Actual target DB was created by this script. The reference file must not
|
|
# be deployed as a second, competing stack while that container exists.
|
|
|
|
|
|
def main():
|
|
global WORK,H2_PASSWORD,TEST_PASSWORD,FINAL_PASSWORD,PG_PIN
|
|
check(sys.version_info >= (3,9),'Python 3.9 oder neuer erforderlich.')
|
|
check(os.geteuid()==0,'Bitte als root auf dem Docker-Host ausfuehren.')
|
|
check(shutil.which('docker'),'Docker fehlt.')
|
|
os.umask(0o077)
|
|
original=inspect(APP)
|
|
check(original['Image']==EXPECTED_IMAGE,'Issuing-Image weicht vom getesteten Image ab.')
|
|
c=original['Config']; hc=original['HostConfig']; env=envmap(c)
|
|
check(c.get('Labels',{}).get('com.docker.compose.service')==APP,'Unerwarteter Compose-Service.')
|
|
mounts=original['Mounts']
|
|
check(len(mounts)==1 and mounts[0].get('Name')==ORIGINAL_VOLUME and mounts[0]['Destination']=='/mnt/persistent',
|
|
'Abweichende Mounts: Konfiguration muss individuell geprueft werden.')
|
|
check(set(original['NetworkSettings']['Networks'])=={'cert-net'},'Unerwartete Issuing-Netzwerke.')
|
|
image=json.loads(docker('image','inspect',EXPECTED_IMAGE))[0]
|
|
check(EJBCA_IMAGE in image.get('RepoDigests',[]),'Image-Digest stimmt nicht.')
|
|
for key in ('Entrypoint','Cmd','User','WorkingDir'):
|
|
check(c.get(key)==image['Config'].get(key),'Benutzerdefinierte Containeroption '+key+' muss separat uebernommen werden.')
|
|
check(not (hc.get('PortBindings') or {}), 'Unerwartete Issuing-Portfreigaben; separat pruefen.')
|
|
check(not hc.get('Privileged') and not hc.get('Devices') and not hc.get('CapAdd') and not hc.get('ExtraHosts'),
|
|
'Benutzerdefinierte Host-Einstellungen muessen separat geprueft werden.')
|
|
url=env.get('DATABASE_JDBC_URL','')
|
|
check(url.startswith('jdbc:h2:/mnt/persistent/'),'Keine erwartete persistente H2-Quelle.')
|
|
base=url.split(';')[0].removeprefix('jdbc:h2:/mnt/persistent/')
|
|
check(re.fullmatch(r'[A-Za-z0-9_.-]+',base) and '..' not in base,'Unerwarteter H2-Dateiname.')
|
|
absent('container',[TEST_DB,TEST_APP,FINAL_DB]); absent('network',[TEST_NETWORK,FINAL_NETWORK])
|
|
absent('volume',[TEST_DB_VOLUME,FINAL_DB_VOLUME,TEST_APP_VOLUME,FINAL_APP_VOLUME])
|
|
BASE.mkdir(mode=0o700,parents=True,exist_ok=True)
|
|
WORK=BASE/('run-'+datetime.datetime.now().strftime('%Y%m%d-%H%M%S')+'-'+secrets.token_hex(3)); WORK.mkdir(mode=0o700)
|
|
for d in ('drivers','classes','recover','restored'): (WORK/d).mkdir()
|
|
(WORK/'original.inspect.json').write_text(json.dumps(original,indent=2))
|
|
H2_PASSWORD=secrets.token_hex(24); TEST_PASSWORD=secrets.token_hex(32); FINAL_PASSWORD=secrets.token_hex(32)
|
|
(WORK/'credentials.private.json').write_text(json.dumps({'h2':H2_PASSWORD,'test_pg':TEST_PASSWORD,'issuing_pg':FINAL_PASSWORD},indent=2))
|
|
log('Arbeitsverzeichnis: '+str(WORK))
|
|
log('1/9 Vorpruefung und Werkzeuge. Root-CA bleibt unberuehrt.')
|
|
for img in (JAVA_IMAGE,PG_IMAGE):
|
|
p=subprocess.run(['docker','image','inspect',img],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
if p.returncode: docker('pull',img,timeout=1800)
|
|
pgimg=json.loads(docker('image','inspect',PG_IMAGE))[0]
|
|
PG_PIN=next((d for d in pgimg.get('RepoDigests',[]) if d.startswith('postgres@')),None)
|
|
check(PG_PIN,'Kein PostgreSQL-Image-Digest verfuegbar.')
|
|
extract_image_files(); prepare_indexes()
|
|
(WORK/'IssuingMigration.java').write_text(MIGRATION_JAVA)
|
|
(WORK/'Inventory.java').write_text(INVENTORY_JAVA)
|
|
docker('run','--rm','--network','none',*mount(WORK,'/work',False),'-w','/work',JAVA_IMAGE,
|
|
'javac','-Xlint:all','-cp','/work/drivers/h2.jar','-d','/work/classes','IssuingMigration.java','Inventory.java')
|
|
log('2/9 Issuing stoppen und Originalvolume sichern. Ab jetzt Ausfallzeit.')
|
|
docker('stop','--time','60',APP)
|
|
check(inspect(APP)['State']['Status']=='exited','Issuing wurde nicht gestoppt.')
|
|
docker('run','--rm','--network','none','--user','0:0','--entrypoint','/bin/sh',
|
|
*mount(ORIGINAL_VOLUME,'/source',True,'volume'),*mount(WORK,'/backup',False),JAVA_IMAGE,
|
|
'-c','set -e; tar --numeric-owner -cpf /backup/original-volume.tar -C /source .')
|
|
docker('cp',APP+':/mnt/persistent/'+base+'.mv.db',str(WORK/'original.mv.db'))
|
|
with tarfile.open(WORK/'original-volume.tar') as tar:
|
|
matches=[m for m in tar.getmembers() if m.name.removeprefix('./')==base+'.mv.db']
|
|
check(len(matches)==1,'H2-Datei fehlt in Volumesicherung.')
|
|
check(hashlib.sha256(tar.extractfile(matches[0]).read()).hexdigest()==digest(WORK/'original.mv.db'),
|
|
'H2-Datei und Volumesicherung stimmen nicht ueberein.')
|
|
(WORK/'original.sha256').write_text(digest(WORK/'original.mv.db')+' original.mv.db\n'+digest(WORK/'original-volume.tar')+' original-volume.tar\n')
|
|
shutil.copyfile(WORK/'original.mv.db',WORK/'recover/ejbcadb.mv.db')
|
|
log('3/9 Recover auf Arbeitskopie; Binaerspalten automatisch korrigieren.')
|
|
docker('run','--rm','--network','none',*mount(WORK/'recover','/recover',False),*mount(WORK/'drivers','/drivers'),
|
|
JAVA_IMAGE,'java','-cp','/drivers/h2.jar','org.h2.tools.Recover','-dir','/recover','-db','ejbcadb')
|
|
check(digest(WORK/'original.mv.db')==digest(WORK/'recover/ejbcadb.mv.db'),'Recover hat die Arbeitskopie veraendert.')
|
|
restored,expected,recover_counts=patch_recover((WORK/'recover/ejbcadb.h2.sql').read_bytes())
|
|
(WORK/'restored/restore.sql').write_text(restored)
|
|
java('Inventory',['restore'],readonly=False)
|
|
counts,binaries=read_inventory()
|
|
check({t:n for t,n in counts.items() if n}==dict(recover_counts),'Rowcounts weichen vom Recover-Dump ab.')
|
|
check(binaries==expected,'Binaerwerte stimmen nicht vollstaendig mit dem Recover-Dump ueberein.')
|
|
log(f'Quelle: {len(counts)} Tabellen, {sum(counts.values())} Zeilen; {sum(expected.values())} Binaerwerte vollstaendig validiert.')
|
|
log('4/9 Isoliertes PostgreSQL-Testziel und JDBC-Migration.')
|
|
docker('network','create','--internal',TEST_NETWORK)
|
|
create_pg(TEST_DB,TEST_DB_VOLUME,TEST_NETWORK,TEST_PASSWORD)
|
|
load_sql(TEST_DB,WORK/'schema.sql'); migrate(counts); load_sql(TEST_DB,WORK/'indexes.sql')
|
|
clean=snapshot(TEST_DB)
|
|
check({t:n for t,(n,h) in clean.items()}=={**counts,'userdatasourcedata':0},'Ziel-Rowcounts weichen ab.')
|
|
(WORK/'clean-counts-and-digests.json').write_text(json.dumps(clean,indent=2))
|
|
log('5/9 EJBCA-Teststart im internen Netz, danach alle Software-Schluessel testen.')
|
|
test_application(env,counts)
|
|
log('6/9 Sauberen Exportstand neu erzeugen; keine Teststart-Zertifikate uebernehmen.')
|
|
sql(TEST_DB,'DROP SCHEMA public CASCADE; CREATE SCHEMA public AUTHORIZATION ejbca;')
|
|
load_sql(TEST_DB,WORK/'schema.sql'); migrate(counts); load_sql(TEST_DB,WORK/'indexes.sql')
|
|
check(snapshot(TEST_DB)==clean,'Neu erzeugter sauberer Stand weicht ab.')
|
|
dump=WORK/'issuing-clean.dump'
|
|
dump.write_bytes(docker('exec',TEST_DB,'pg_dump','-U','ejbca','-d','ejbcadb','-Fc',timeout=3600))
|
|
check(dump.stat().st_size>0,'Leerer Dump.')
|
|
(WORK/'dump-contents.txt').write_bytes(docker('exec','-i',TEST_DB,'pg_restore','--list',data=dump.read_bytes()))
|
|
(WORK/'issuing-clean.dump.sha256').write_text(digest(dump)+' issuing-clean.dump\n')
|
|
log('7/9 Neues endgueltiges Issuing-PostgreSQL-Ziel erstellen und Dump wiederherstellen.')
|
|
docker('network','create','--internal',FINAL_NETWORK)
|
|
create_pg(FINAL_DB,FINAL_DB_VOLUME,FINAL_NETWORK,FINAL_PASSWORD)
|
|
docker('exec','-i',FINAL_DB,'pg_restore','-U','ejbca','-d','ejbcadb','--exit-on-error','--single-transaction',data=dump.read_bytes(),timeout=3600)
|
|
check(snapshot(FINAL_DB)==clean,'Wiederhergestelltes Ziel stimmt nicht vollstaendig ueberein.')
|
|
log('8/9 Separates App-Volume und fertige Compose-Dateien vorbereiten.')
|
|
copy_volume(FINAL_APP_VOLUME); write_compose(original,envmap(image['Config']))
|
|
# Validate syntax using Docker Compose if available; no deployment is performed.
|
|
p=subprocess.run(['docker','compose','version'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
if p.returncode==0:
|
|
docker('compose','-f',str(WORK/'docker-compose.issuing.yml'),'config','--quiet')
|
|
report={'source_tables':len(counts),'source_rows':sum(counts.values()),'binary_values_checked':sum(expected.values()),
|
|
'indexes':44,'dump_sha256':digest(dump),'source_sha256':digest(WORK/'original.mv.db'),
|
|
'production_database':FINAL_DB,'app_deployed':False,'root_resources_modified':False}
|
|
(WORK/'report.json').write_text(json.dumps(report,indent=2))
|
|
log('9/9 Vorbereitung erfolgreich. Issuing-App ist noch gestoppt; PostgreSQL-Ziel ist bereit.')
|
|
print('\nNEUES ISSUING-POSTGRESQL-PASSWORT (lokal vertraulich aufbewahren):\n'+FINAL_PASSWORD)
|
|
print('\nCOMPOSE FUER DEN BESTEHENDEN PORTAINER-STACK ejbca-issuing:\n')
|
|
print((WORK/'docker-compose.issuing.yml').read_text())
|
|
print('Dateien: '+str(WORK))
|
|
print('Jetzt diese Compose-Datei in Portainer im vorhandenen Issuing-Stack einsetzen und aktualisieren.')
|
|
print('Die DATABASE-REFERENCE-Datei ist nur eine Dokumentation des bereits laufenden DB-Containers; nicht zusaetzlich deployen.')
|
|
print('Nach Deployment: HTTP-Bereitschaft, Adminoberflaeche und Schluessel im echten Issuing-Container pruefen.')
|
|
print('Die ausgegebene Compose-Datei enthaelt Geheimnisse. Nicht ungeschwaerzt in Chats posten.')
|
|
|
|
if __name__=='__main__':
|
|
try: main()
|
|
except (Exception,KeyboardInterrupt) as e:
|
|
print('\nABBRUCH: '+(str(e) if isinstance(e,MigrationError) else type(e).__name__),file=sys.stderr)
|
|
if WORK:
|
|
print('Sicherungen und Diagnose: '+str(WORK),file=sys.stderr)
|
|
# Stop only the explicitly named test app, never the source or root app.
|
|
try:
|
|
owned=inspect(TEST_APP)['Config'].get('Labels',{}).get('issuing-migration.run')==str(WORK)
|
|
if owned: subprocess.run(['docker','stop','--time','30',TEST_APP],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
except Exception: pass
|
|
print('Nicht blind erneut starten. Es werden keine Ressourcen automatisch geloescht.',file=sys.stderr)
|
|
print('Die urspruengliche Issuing-Compose-Konfiguration und ihr H2-Volume wurden nicht geaendert.',file=sys.stderr)
|
|
sys.exit(1)
|