Importing old Silence Texts/SMS to QKSMS/Android

#! /usr/bin/python3

# SMS Import / Export: a simple Android app for importing and exporting SMS and MMS messages,
# call logs, and contacts, from and to JSON / NDJSON files.
#
# Copyright (c) 2023 Thomas More
#
# This file is part of SMS Import / Export.
#
# SMS Import / Export is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# SMS Import / Export is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with SMS Import / Export.  If not, see <https://www.gnu.org/licenses/>

# This utility converts message in Silence (https://silence.im/) XML format to SMS I/E 'v2' format
# Usage: 'silence-convert.py <silence-xxx.xml>'
# This will read messages from <silence-xxx.xml> and write them to <silence-xxx.zip>.

import sys
import json
import xml.etree.ElementTree as ET
import zipfile
import os
import re # Module for regular expressions

# --- START OF CLEANUP FUNCTIONS ---

def is_invalid_xml_char_code(code):
    """
    Checks if a given decimal Unicode code point is invalid in XML 1.0.
    Invalid are: Control characters 0-8, 11-12, 14-31 and the surrogate range D800-DFFF (55296-57343).
    """
    # Control character ranges (except Tab=9, LF=10, CR=13)
    if 0 <= code <= 8: return True
    if 11 <= code <= 12: return True
    if 14 <= code <= 31: return True
    # Surrogate range
    if 55296 <= code <= 57343: return True
    # Otherwise, the code is valid
    return False

def check_and_replace_numeric_ref(match):
    """
    Callback function for re.sub. Checks a found numeric reference.
    Returns an empty string if it's invalid, otherwise the original.
    """
    try:
        if match.group(1): # Decimal match (group 1)
            code = int(match.group(1))
        elif match.group(2): # Hex match (group 2)
            code = int(match.group(2), 16)
        else:
            # Should not happen with the used regex, but better safe than sorry
             return match.group(0)

        if is_invalid_xml_char_code(code):
            # Invalid code -> replace with empty string
            # print(f"DEBUG: Removing invalid reference: {match.group(0)}") # Optional: Debugging
            return ""
        else:
            # Valid code -> keep original reference
            return match.group(0)
    except ValueError:
        # If something goes wrong with int() -> keep original
        return match.group(0)
    except Exception:
        # Other unexpected errors -> keep original
        return match.group(0)

def clean_xml_content(xml_string):
    """
    Cleans an XML string more comprehensively:
    1. Removes raw XML 1.0 invalid control characters.
    2. Finds all numeric character references (dec/hex) and removes
       those pointing to invalid code points (control characters or surrogates).
    """

    # 1. Regex to find and remove raw invalid control characters
    try:
        raw_invalid_chars_pattern = re.compile(r'[\x00-\x08\x0B\x0C\x0E-\x1F]')
        cleaned_string = raw_invalid_chars_pattern.sub('', xml_string)
        print(f"  (Step 1: Raw control characters removed)")
    except Exception as e:
        print(f"  (Error in Step 1 - Raw control characters: {e})")
        cleaned_string = xml_string # In case of error, continue with original string

    # 2. Regex to find *all* numeric character references (decimal OR hexadecimal)
    #    Group 1 captures the decimal value, group 2 the hex value
    try:
        numeric_ref_pattern = re.compile(r"&(?:#([0-9]+)|#x([0-9A-Fa-f]+));", re.IGNORECASE)

        # Use re.sub with the callback function to check each reference
        final_cleaned_string = numeric_ref_pattern.sub(check_and_replace_numeric_ref, cleaned_string)
        print(f"  (Step 2: Invalid numeric references (incl. surrogates) checked & potentially removed)")
    except Exception as e:
        print(f"  (Error in Step 2 - Checking numeric references: {e})")
        final_cleaned_string = cleaned_string # In case of error, continue with the string from step 1

    return final_cleaned_string

# --- END OF CLEANUP FUNCTIONS ---


# --- Main script logic ---

# Check if a filename was passed as an argument
if len(sys.argv) < 2:
    print("Error: Please provide the name of the input XML file.")
    print("Usage: python3 silence-convert.py <silence-xxx.xml>")
    sys.exit(1)

# Assign the first argument to the input_file variable
input_file = sys.argv[1]

# Read the content of the XML file
try:
    print(f"Reading XML file: {input_file}")
    # Try with UTF-8
    with open(input_file, 'r', encoding='utf-8') as f:
        xml_content = f.read()
except FileNotFoundError:
    print(f"Error: Input file not found: {input_file}")
    sys.exit(1)
except UnicodeDecodeError:
    try:
        print("Warning: Could not read file as UTF-8, trying latin-1...")
        with open(input_file, 'r', encoding='latin-1') as f:
            xml_content = f.read()
    except Exception as e:
        print(f"Error reading file {input_file} with fallback encoding: {e}")
        sys.exit(1)
except Exception as e:
    print(f"An unexpected error occurred while reading the file: {e}")
    sys.exit(1)

# Clean the XML content from invalid characters/references
print("Cleaning XML content (comprehensive attempt)...")
cleaned_xml_content = clean_xml_content(xml_content)

# Parse the cleaned XML string
print("Parsing cleaned XML content...")
try:
    # Use ET.fromstring as we have the content as a string
    root = ET.fromstring(cleaned_xml_content)
    smses = root # The root element is now 'root'
except ET.ParseError as e:
    # If parsing still fails, there might be another XML issue
    print(f"XML parsing failed even after comprehensive cleaning: {e}")
    print("The XML file might contain other errors or be severely corrupted.")
    # Debug info for the problematic line
    try:
        problem_line_num = e.position[0] # Get line number from the error
        lines = cleaned_xml_content.splitlines()
        if 0 < problem_line_num <= len(lines):
            print(f"--- Problematic line (No. {problem_line_num}) in *cleaned* content (excerpt): ---")
            # Show line before, the line itself, line after
            start = max(0, problem_line_num - 2)
            end = min(len(lines), problem_line_num + 1)
            for i in range(start, end):
                 print(f"{i+1}: {lines[i][:200]}") # Show max 200 characters per line
            print("--- End of excerpt ---")
        else:
             print("(Could not extract problematic line)")
    except Exception as debug_e:
        print(f"(Error extracting problematic line: {debug_e})")
    sys.exit(1)
except Exception as e:
    print(f"An unexpected error occurred while parsing the XML: {e}")
    sys.exit(1)

# Define the output filename (requires input_file)
output_file = input_file[:-3] + 'zip' if input_file.lower().endswith('.xml') else input_file + '.zip'

# Process messages and write ZIP
print(f"Processing messages and writing to ZIP file: {output_file}")
try:
    with zipfile.ZipFile(output_file, mode='w', compression=zipfile.ZIP_DEFLATED) as messages_zip:
        message_count = 0
        messages_ndjson = [] # Ensure the list is initialized here
        # Ensure 'smses' is iterable (the root element itself contains the 'sms'/'mms' tags)
        for sms in smses: # Iterate over the child elements of the root element
            # Convert the element's attributes into a dictionary
            message_data = dict(sms.attrib) # Use sms.attrib for XML attributes
            messages_ndjson.append(json.dumps(message_data) + '\n')
            message_count += 1

        if not messages_ndjson:
             print("Warning: No messages found or processed in the XML.")
        else:
            # Write the collected NDJSON lines to the file inside the ZIP
            messages_zip.writestr('messages.ndjson', ''.join(messages_ndjson))
            print(f"{message_count} messages successfully processed and written to {output_file}.")

except Exception as e:
    print(f"Error processing messages or writing the ZIP file: {e}")
    sys.exit(1)

print("Script finished.")