1#************************************************************** 2# 3# Licensed to the Apache Software Foundation (ASF) under one 4# or more contributor license agreements. See the NOTICE file 5# distributed with this work for additional information 6# regarding copyright ownership. The ASF licenses this file 7# to you under the Apache License, Version 2.0 (the 8# "License"); you may not use this file except in compliance 9# with the License. You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, 14# software distributed under the License is distributed on an 15# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16# KIND, either express or implied. See the License for the 17# specific language governing permissions and limitations 18# under the License. 19# 20#************************************************************** 21 22# usage: python -m aootest.checkers.pkgcheck <outdir> <pkg.json> (needs AOO_JING and AOO_SCHEMA_DIR; tests call check()) 23# ODF 1.2 Part 3 2.2.1, 3.2, 3.3 and 3.4.1 on every package in <outdir>. Each finding is 24# "shall: [rule] ..." or "should: [rule] ...". Schema validity of the manifest is validate.py's job. 25import sys, os, re, zipfile, json, struct, subprocess, shutil, warnings 26from xml.etree import ElementTree as ET 27 28DSIG_SCHEMA_FILE = "OpenDocument-v1.2-os-dsig-schema.rng" 29MANI = "{urn:oasis:names:tc:opendocument:xmlns:manifest:1.0}" 30DSIG = "{urn:oasis:names:tc:opendocument:xmlns:digitalsignature:1.0}" 31ERR = re.compile(r"^(.*?):(\d+):(\d+): (error|fatal|warning): (.*)$") 32 33 34def check_package(path, work, jing, dsig_schema): 35 """Returns [(entry, level, rule, message)] for one package.""" 36 out = [] 37 38 def add(entry, level, rule, msg): 39 out.append((entry, level, rule, msg)) 40 z = zipfile.ZipFile(path) 41 infos = z.infolist() 42 names = [i.filename for i in infos] 43 for n in sorted({n for n in names if names.count(n) > 1}): 44 add(n, "shall", "zip-duplicate", "zip entry name occurs more than once") 45 for i in infos: 46 if i.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED): 47 add(i.filename, "shall", "zip-method", "compression method %d is neither STORED nor DEFLATED (2.2.1 A)" % i.compress_type) 48 if i.filename.endswith("/"): 49 add(i.filename, "should", "zip-dir-entry", "directories have no zip entries (4.3)") 50 files = {n for n in names if not n.endswith("/")} 51 52 # mimetype (3.3) 53 mime = None 54 if "mimetype" not in files: 55 add("mimetype", "should", "mimetype-missing", "no mimetype file (2.2.1 C)") 56 else: 57 mi = z.getinfo("mimetype") 58 first = min(infos, key=lambda i: i.header_offset) 59 if first.filename != "mimetype" or mi.header_offset != 0: 60 add("mimetype", "shall", "mimetype-first", "mimetype is not the first file of the zip (3.3)") 61 with open(path, "rb") as f: 62 f.seek(mi.header_offset) 63 hdr = f.read(30) 64 if len(hdr) == 30 and hdr[:4] == b"PK\x03\x04": 65 method, extralen = struct.unpack("<H", hdr[8:10])[0], struct.unpack("<H", hdr[28:30])[0] 66 if method != zipfile.ZIP_STORED: 67 add("mimetype", "shall", "mimetype-compressed", "mimetype is compressed, local header method %d (3.3)" % method) 68 if extralen: 69 add("mimetype", "shall", "mimetype-extra", "mimetype has a %d byte extra field in its local header (3.3)" % extralen) 70 else: 71 add("mimetype", "shall", "mimetype-first", "no local file header at the recorded offset") 72 if mi.compress_type != zipfile.ZIP_STORED: 73 add("mimetype", "shall", "mimetype-compressed", "mimetype is compressed, central directory method %d (3.3)" % mi.compress_type) 74 raw = z.read("mimetype") 75 try: 76 mime = raw.decode("ascii") 77 except UnicodeDecodeError: 78 add("mimetype", "shall", "mimetype-ascii", "mimetype content is not ASCII (3.3)") 79 80 # META-INF contents (2.2.1 D, E) 81 for n in sorted(files): 82 if n.startswith("META-INF/") and n != "META-INF/manifest.xml" and "signatures" not in n.rsplit("/", 1)[-1]: 83 add(n, "shall", "meta-inf-file", "file other than manifest.xml and signature files in META-INF (2.2.1 E)") 84 85 # manifest (2.2.1 B, 3.2) 86 entries = None 87 if "META-INF/manifest.xml" not in files: 88 add("META-INF/manifest.xml", "shall", "manifest-missing", "no META-INF/manifest.xml (2.2.1 B)") 89 else: 90 try: 91 root = ET.fromstring(z.read("META-INF/manifest.xml")) 92 if root.tag != MANI + "manifest": 93 add("META-INF/manifest.xml", "shall", "manifest-root", "root element is not manifest:manifest (2.2.1 B.2)") 94 else: 95 entries = [e for e in root if e.tag == MANI + "file-entry"] 96 except ET.ParseError as e: 97 add("META-INF/manifest.xml", "shall", "manifest-malformed", "not well-formed: %s (2.2.1 B.1)" % e) 98 if entries is not None: 99 paths = [e.get(MANI + "full-path") for e in entries] 100 for p in sorted({p for p in paths if paths.count(p) > 1}, key=str): 101 add("META-INF/manifest.xml", "shall", "manifest-duplicate", "more than one file-entry for %s (3.2)" % p) 102 for p in paths: 103 if p in ("META-INF/manifest.xml", "mimetype"): 104 add("META-INF/manifest.xml", "shall", "manifest-self", "file-entry for %s (3.2)" % p) 105 elif p and p.endswith("/") and p != "/": 106 if p + "content.xml" not in files and p + "styles.xml" not in files: 107 add("META-INF/manifest.xml", "should", "manifest-dir-entry", "file-entry for %s, a directory without a document (4.3)" % p) 108 elif p not in files and p != "/": 109 add("META-INF/manifest.xml", "should", "manifest-dangling", "file-entry for %s, which is not in the package (3.2)" % p) 110 for n in sorted(files): 111 if n != "mimetype" and not n.startswith("META-INF/") and n not in paths: 112 add(n, "shall", "manifest-coverage", "no file-entry in the manifest (3.2)") 113 root_entry = [e for e in entries if e.get(MANI + "full-path") == "/"] 114 if "mimetype" in files and not root_entry: 115 add("META-INF/manifest.xml", "shall", "manifest-root-entry", 'no file-entry for "/" although the package has a mimetype file (3.2)') 116 for e in root_entry: 117 mt = e.get(MANI + "media-type") 118 if "mimetype" not in files: 119 add("mimetype", "shall", "mimetype-missing", 'file-entry for "/" but no mimetype file (3.3)') 120 elif mime is not None and mt != mime: 121 add("mimetype", "shall", "mimetype-mismatch", 'mimetype is "%s" but the "/" media-type is "%s" (3.3)' % (mime, mt)) 122 # encrypted entries (3.4.1) 123 for e in entries: 124 if e.find(MANI + "encryption-data") is None: 125 continue 126 p = e.get(MANI + "full-path") 127 size = e.get(MANI + "size") 128 if size is None or not re.fullmatch(r"[0-9]+", size): 129 add(p, "shall", "encrypted-size", "encrypted entry without a valid manifest:size (3.4.1, 4.8.13)") 130 if p in files and z.getinfo(p).compress_type != zipfile.ZIP_STORED: 131 add(p, "shall", "encrypted-method", "encrypted entry is not flagged STORED in the zip (3.4.1)") 132 133 # signature files (2.2.1 D) 134 sigs = sorted(n for n in files if n.startswith("META-INF/") and "signatures" in n.rsplit("/", 1)[-1]) 135 for n in sigs: 136 try: 137 root = ET.fromstring(z.read(n)) 138 except ET.ParseError as e: 139 add(n, "shall", "signature-malformed", "not well-formed: %s (2.2.1 D.1)" % e) 140 continue 141 if root.tag != DSIG + "document-signatures": 142 add(n, "shall", "signature-root", "root element is not dsig:document-signatures (2.2.1 D.2)") 143 continue 144 p = os.path.join(work, os.path.basename(path), n) 145 os.makedirs(os.path.dirname(p), exist_ok=True) 146 open(p, "wb").write(z.read(n)) 147 r = subprocess.run(jing + [dsig_schema, p], capture_output=True, text=True) 148 for line in (r.stdout + r.stderr).splitlines(): 149 m = ERR.match(line) 150 if m: 151 add(n, "shall", "signature-schema", "%s:%s %s (2.2.1 D.3)" % (m.group(2), m.group(3), m.group(5))) 152 elif line.strip(): 153 add(n, "shall", "signature-schema", line.strip()) 154 return out 155 156 157def check(outdir, workdir, jing, schema_dir): 158 """Returns the result that main() writes as <pkg.json>. `workdir` must be empty or absent; nothing is deleted.""" 159 work = os.path.abspath(workdir) 160 if os.path.isdir(work) and os.listdir(work): 161 raise ValueError("pkgcheck: workdir is not empty: " + work) 162 dsig_schema = os.path.join(schema_dir, DSIG_SCHEMA_FILE) 163 errors, bad_zip = [], [] 164 pkgs = sorted(p for p in os.listdir(outdir) if not p.startswith(".")) 165 for pkg in pkgs: 166 try: 167 found = check_package(os.path.join(outdir, pkg), work, jing, dsig_schema) 168 except zipfile.BadZipFile: 169 bad_zip.append(pkg); continue 170 for entry, level, rule, msg in found: 171 errors.append([pkg + "/" + entry, 0, 0, "%s: [%s] %s" % (level, rule, msg)]) 172 return {"packages": pkgs, "bad_zip": bad_zip, "errors": errors} 173 174 175def main(argv=None): 176 outdir, pkgjson = sys.argv[1:3] if argv is None else argv 177 res = check(outdir, pkgjson[:-5] + ".xml", ["java", "-jar", os.environ["AOO_JING"], "-i"], os.environ["AOO_SCHEMA_DIR"]) 178 json.dump(res, open(pkgjson, "w")) 179 errors, pkgs, bad_zip = res["errors"], res["packages"], res["bad_zip"] 180 shall = [e for e in errors if e[3].startswith("shall:")] 181 print(f"package check: {len(shall)} shall and {len(errors) - len(shall)} should findings in " 182 f"{len({e[0].split('/')[0] for e in errors})} of {len(pkgs)} packages, bad zips: {bad_zip}") 183 rules = {} 184 for e in errors: 185 k = e[3].split("]")[0] + "]" 186 rules[k] = rules.get(k, 0) + 1 187 for k in sorted(rules): 188 print(f" {rules[k]:6d} {k}") 189 190 191if __name__ == "__main__": 192 main() 193