diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f231449..ecc0e7bb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,16 +83,11 @@ jobs: echo "Downloading latest 'eu-lotl.xml' file" wget "https://ec.europa.eu/tools/lotl/eu-lotl.xml" -P $TSL_FILES_DIRECTORY - if [ -f "$TSL_FILES_DIRECTORY/eu-lotl.xml" ]; then - echo "Getting TSL (local LOTL)" - python3 scripts/tsl_xml_downloader.py --tslFile="$TSL_FILES_DIRECTORY/eu-lotl.xml" --tslTestFile="$TSL_FILES_DIRECTORY/$(basename 'https://ec.europa.eu/tools/lotl/eu-lotl.xml')" --countries="EE" --isDevBuild="False" - - echo "Finished downloading TSL" - else - echo "Getting TSL (remote LOTL)" - python3 scripts/tsl_xml_downloader.py --tslTestFile="$TSL_FILES_DIRECTORY/$(basename 'https://ec.europa.eu/tools/lotl/eu-lotl.xml')" --countries="EE" --isDevBuild="False" - echo "Finished downloading TSL" - fi + echo "Getting TSL" + python3 scripts/tsl_xml_downloader.py \ + --tslFile="$TSL_FILES_DIRECTORY/eu-lotl.xml" \ + --members="EE" + echo "Finished downloading TSL" echo "Moving TSLs to app's TSL files directory" mv -v scripts/TSL/* $TSL_FILES_DIRECTORY diff --git a/codemagic.yaml b/codemagic.yaml index 0393112f..6be1a800 100644 --- a/codemagic.yaml +++ b/codemagic.yaml @@ -33,9 +33,9 @@ workflows: defaultCentralConfigurationUrl_input: description: Default Central Configuration URL type: choice - default: http://10.1.24.31/config/mobile + default: http://ria.test/config/mobile options: - - http://10.1.24.31/config/mobile + - http://ria.test/config/mobile - https://id.eesti.ee - https://id-test.eesti.ee - https://id.test.eesti.ee @@ -46,9 +46,9 @@ workflows: defaultCentralConfigurationTslUrl_input: description: Default Central Configuration TSL URL type: choice - default: http://10.1.24.31/tsl/trusted-test-mp.xml + default: http://ria.test/tsl/trusted-test-mp.xml options: - - http://10.1.24.31/tsl/trusted-test-mp.xml + - http://ria.test/tsl/trusted-test-mp.xml - https://ec.europa.eu/tools/lotl/eu-lotl.xml - https://open-eid.github.io/test-TL/tl-mp-test-EE.xml memberTsl_input: @@ -73,12 +73,20 @@ workflows: - *get_google_services_plist - name: "Setup config and TSL files" script: | + set -e -o pipefail + + TSL_URL='${{ inputs.defaultCentralConfigurationTslUrl_input }}' + MEMBER_TSLS='${{ inputs.memberTsl_input }}' + export CONFIG_DIRECTORY="$RESOURCES_DIRECTORY/config" export TSL_FILES_DIRECTORY="$RESOURCES_DIRECTORY/tslFiles" + export TSL_BASE_DIRECTORY="$CM_BUILD_DIR/tslBaseFiles" + export CONFIGURED_TSL="$(basename "$TSL_URL")" # Create config, TSL folders and mock folders for each module mkdir -p $CONFIG_DIRECTORY mkdir -p $TSL_FILES_DIRECTORY + mkdir -p $TSL_BASE_DIRECTORY # SPM packages must exist and need at least 1 file. Mock files are generated in 'Run tests' step for module in CommonsLib ConfigLib WebEidLib CryptoLib IdCardLib LibdigidocLib MobileIdLib SmartIdLib UtilsLib; do @@ -93,47 +101,107 @@ workflows: # Download latest 'eu-lotl.xml' file" echo "Downloading latest 'eu-lotl.xml' file" - wget "https://ec.europa.eu/tools/lotl/eu-lotl.xml" -P $TSL_FILES_DIRECTORY + wget -q "https://ec.europa.eu/tools/lotl/eu-lotl.xml" -O "$TSL_BASE_DIRECTORY/eu-lotl.xml" # Get repo.ria values from environmental variables, as CodeMagic doesn't have access to repo.ria domain - if [[ "${{ inputs.defaultCentralConfigurationTslUrl_input }}" == *"10.1.24.31"* ]]; then + if [[ "$TSL_URL" == *"ria.test"* ]]; then echo "Using repo.ria configuration" echo $REPO_RIA_DEFAULT_PROPERTIES | base64 --decode > $CONFIG_DIRECTORY/configuration.properties echo $REPO_RIA_CONFIG | base64 --decode > $CONFIG_DIRECTORY/default-config.json echo $REPO_RIA_PUB_KEY | base64 --decode > $CONFIG_DIRECTORY/default-config.ecpub echo $REPO_RIA_ECC | base64 --decode > $CONFIG_DIRECTORY/default-config.ecc - echo $REPO_RIA_TRUSTED_TEST_MP | base64 --decode > $TSL_FILES_DIRECTORY/trusted-test-mp.xml - echo $REPO_RIA_TL_MP_TEST_EE | base64 --decode > $TSL_FILES_DIRECTORY/tl-mp-test-EE.xml - # EE_T.xml is too large to use in environmental variable. Using compressed variant - echo $REPO_RIA_EE_T_TSL | base64 --decode | gunzip > $TSL_FILES_DIRECTORY/EE_T.xml + echo $REPO_RIA_TRUSTED_TEST_MP | base64 --decode > "$TSL_BASE_DIRECTORY/trusted-test-mp.xml" + + for configFile in "$CONFIG_DIRECTORY/configuration.properties" "$CONFIG_DIRECTORY/default-config.json" \ + "$CONFIG_DIRECTORY/default-config.ecpub" "$CONFIG_DIRECTORY/default-config.ecc" \ + "$TSL_BASE_DIRECTORY/trusted-test-mp.xml"; do + if [ ! -s "$configFile" ]; then + echo "ERROR: $configFile is missing or empty" + exit 1 + fi + done else echo "Not using repo.ria configuration" + + if [ ! -f "$TSL_BASE_DIRECTORY/$CONFIGURED_TSL" ]; then + echo "Downloading '$CONFIGURED_TSL' file" + wget -q "$TSL_URL" -O "$TSL_BASE_DIRECTORY/$CONFIGURED_TSL" + fi fi - if [ -f "$TSL_FILES_DIRECTORY/eu-lotl.xml" ]; then - echo "Getting TSL (local LOTL)" - # Repo.ria TSL files already in assets folder, doesn't need to download separately anymore - if [[ "${{ inputs.defaultCentralConfigurationTslUrl_input }}" == *"10.1.24.31"* ]]; then - python3 scripts/tsl_xml_downloader.py --tslFile="$TSL_FILES_DIRECTORY/eu-lotl.xml" --countries="${{ inputs.memberTsl_input }}" --isDevBuild="True" + echo "Writing requested test member TSLs from environment variables" + MEMBERS_TO_DOWNLOAD="" + for member in $(echo "$MEMBER_TSLS" | tr ';' ' '); do + if [[ ! "$member" =~ ^[A-Za-z0-9_]+$ ]]; then + echo "Invalid TSL member code: '$member'" + exit 1 + fi + + if [[ "$member" != *_T ]]; then + echo "Member '$member' is downloaded from a base file" + MEMBERS_TO_DOWNLOAD="$MEMBERS_TO_DOWNLOAD;$member" + continue + fi + + variable_name="REPO_RIA_${member}_TSL" + variable_value="$(printenv "$variable_name" || true)" + + if [ -z "$variable_value" ]; then + echo "Test member '$member' was requested but $variable_name is not set" + exit 1 + fi + + echo "Writing $member.xml from $variable_name" + member_file="$TSL_FILES_DIRECTORY/$member.xml" + printf '%s' "$variable_value" | base64 --decode > "$member_file.raw" + if gzip -t "$member_file.raw" 2> /dev/null; then + gunzip -c "$member_file.raw" > "$member_file" else - python3 scripts/tsl_xml_downloader.py --tslFile="$TSL_FILES_DIRECTORY/eu-lotl.xml" --tslTestFile="$TSL_FILES_DIRECTORY/$(basename ${{ inputs.defaultCentralConfigurationTslUrl_input }})" --countries="${{ inputs.memberTsl_input }}" --isDevBuild="True" + cp "$member_file.raw" "$member_file" fi + rm -f "$member_file.raw" + + if [ ! -s "$member_file" ]; then + echo "ERROR: $member_file is missing or empty" + exit 1 + fi + done + MEMBERS_TO_DOWNLOAD="${MEMBERS_TO_DOWNLOAD#;}" - echo "Finished downloading TSL" + echo "Getting TSL" + python3 scripts/tsl_xml_downloader.py \ + --tslFile="$TSL_BASE_DIRECTORY/eu-lotl.xml" \ + --tslFile="$TSL_BASE_DIRECTORY/trusted-test-mp.xml" \ + --members="$MEMBERS_TO_DOWNLOAD" + echo "Finished downloading TSL" + + if [ -n "$(ls -A scripts/TSL 2> /dev/null)" ]; then + echo "Moving TSLs to app's TSL files directory" + mv -v scripts/TSL/* $TSL_FILES_DIRECTORY + echo "Done moving TSLs" else - echo "Getting TSL (remote LOTL)" - python3 scripts/tsl_xml_downloader.py --tslTestFile="$TSL_FILES_DIRECTORY/$(basename 'https://ec.europa.eu/tools/lotl/eu-lotl.xml')" --countries="${{ inputs.memberTsl_input }}" --isDevBuild="True" - echo "Finished downloading TSL" + echo "No downloaded TSLs to move" fi - echo "Moving TSLs to app's TSL files directory" - mv -v scripts/TSL/* $TSL_FILES_DIRECTORY - echo "Done moving TSLs" + if [[ "$TSL_URL" == *"ria.test"* ]]; then + echo "Adding repo.ria TSL to app's TSL files directory" + cp -v "$TSL_BASE_DIRECTORY/trusted-test-mp.xml" "$TSL_FILES_DIRECTORY/" + else + echo "Adding eu-lotl.xml to app's TSL files directory" + cp -v "$TSL_BASE_DIRECTORY/eu-lotl.xml" "$TSL_FILES_DIRECTORY/" + + if [ "$CONFIGURED_TSL" != "eu-lotl.xml" ]; then + echo "Adding $CONFIGURED_TSL to app's TSL files directory" + cp -v "$TSL_BASE_DIRECTORY/$CONFIGURED_TSL" "$TSL_FILES_DIRECTORY/" + fi + fi + + echo "TSL files bundled in resources: $(ls "$TSL_FILES_DIRECTORY" | tr '\n' ' ')" - *get_app_version - name: Fetch default configuration script: | # Not running 'SetupConfiguration' for repo.ria as necessary files are already setup - if [[ ! "${{ inputs.defaultCentralConfigurationTslUrl_input }}" == *"10.1.24.31"* ]]; then + if [[ ! "${{ inputs.defaultCentralConfigurationTslUrl_input }}" == *"ria.test"* ]]; then echo "Fetching default configuration" python3 scripts/setup_configuration.py ${{ inputs.defaultCentralConfigurationUrl_input }} ${{ inputs.defaultCentralConfigurationUpdateInterval_input }} ${{ inputs.defaultCentralConfigurationTslUrl_input }} else @@ -158,7 +226,7 @@ workflows: script: | # Skip fetching configuration for repo.ria as necessary files are already setup REPO_RIA_CONF_FLAGS="" - if [[ "${{ inputs.defaultCentralConfigurationTslUrl_input }}" == *"10.1.24.31"* ]]; then + if [[ "${{ inputs.defaultCentralConfigurationTslUrl_input }}" == *"ria.test"* ]]; then REPO_RIA_CONF_FLAGS="DEFAULT_CENTRAL_CONFIGURATION_URL='' DEFAULT_CENTRAL_CONFIGURATION_UPDATE_INTERVAL=4 DEFAULT_CENTRAL_CONFIGURATION_TSL_URL=''" fi @@ -232,6 +300,8 @@ workflows: - *get_google_services_plist - name: "Setup config and TSL files" script: | + set -e -o pipefail + export CONFIG_DIRECTORY="$RESOURCES_DIRECTORY/config" export TSL_FILES_DIRECTORY="$CONFIG_DIRECTORY/tslFiles.bundle" @@ -251,22 +321,19 @@ workflows: # Download latest 'eu-lotl.xml' file" echo "Downloading latest 'eu-lotl.xml' file" - wget "https://ec.europa.eu/tools/lotl/eu-lotl.xml" -P $TSL_FILES_DIRECTORY + wget -q "https://ec.europa.eu/tools/lotl/eu-lotl.xml" -O "$TSL_FILES_DIRECTORY/eu-lotl.xml" - if [ -f "$TSL_FILES_DIRECTORY/eu-lotl.xml" ]; then - echo "Getting TSL (local LOTL)" - python3 scripts/tsl_xml_downloader.py --tslFile="$TSL_FILES_DIRECTORY/eu-lotl.xml" --tslTestFile="$TSL_FILES_DIRECTORY/$(basename 'https://ec.europa.eu/tools/lotl/eu-lotl.xml')" --countries="EE" --isDevBuild="False" - - echo "Finished downloading TSL" - else - echo "Getting TSL (remote LOTL)" - python3 scripts/tsl_xml_downloader.py --tslTestFile="$TSL_FILES_DIRECTORY/$(basename 'https://ec.europa.eu/tools/lotl/eu-lotl.xml')" --countries="EE" --isDevBuild="False" - echo "Finished downloading TSL" - fi + echo "Getting TSL" + python3 scripts/tsl_xml_downloader.py \ + --tslFile="$TSL_FILES_DIRECTORY/eu-lotl.xml" \ + --members="EE" + echo "Finished downloading TSL" echo "Moving TSLs to app's TSL files directory" mv -v scripts/TSL/* $TSL_FILES_DIRECTORY echo "Done moving TSLs" + + echo "TSL files bundled in resources: $(ls "$TSL_FILES_DIRECTORY" | tr '\n' ' ')" - name: Fetch default configuration script: | echo "Fetching default configuration" diff --git a/scripts/tsl_xml_downloader.py b/scripts/tsl_xml_downloader.py index 3792f9c8..f53f3c9b 100644 --- a/scripts/tsl_xml_downloader.py +++ b/scripts/tsl_xml_downloader.py @@ -1,131 +1,121 @@ -#!/usr/bin/python +#!/usr/bin/env python3 -import urllib.request -import ssl import argparse -import os +import urllib.request +from pathlib import Path from xml.dom import minidom -class TSLDownloader: - def __init__(self): - self.mimetypeFound = False - - def parse_arguments(self): - """Parse command-line arguments.""" - parser = argparse.ArgumentParser(description='TSL XML Downloader') - parser.add_argument('--tslFile', default='eu-lotl.xml', type=str, help='TSL file location') - parser.add_argument('--tslTestFile', type=str, help='Test TSL file location') - parser.add_argument('--countries', required=True, type=str, help='Countries, e.g., "EE;EE_T"') - parser.add_argument('--isDevBuild', required=True, type=str, help='Indicates if build is dev version, e.g., "True"') - return parser.parse_args() - - def load_xml(self, tsl_file_location, tsl_test_file_location, is_test_file): - """Load the appropriate XML file based on country and test file presence.""" - if is_test_file: - if tsl_test_file_location is None or tsl_test_file_location == tsl_file_location: - return None - return minidom.parse(tsl_test_file_location) - - if os.path.isfile(tsl_file_location): - return minidom.parse(tsl_file_location) - return minidom.parse(tsl_test_file_location) - - def download_tsl(self, tsl_url, country_code, is_dev_build, processed_urls): - """Download TSL data from the given URL and save it.""" - # Check if the URL has already been processed - if tsl_url in processed_urls: - print(f"Skipping already processed URL: {tsl_url}") - return - - print(f"{country_code} TSL-URL: {tsl_url}") - - # Get data from TSL-URL - response = urllib.request.urlopen(tsl_url).read() - - # Decode the response from bytes to string - response_str = response.decode('utf-8') - - # Get the directory of the current script - script_directory = os.path.dirname(os.path.abspath(__file__)) - - # Define the TSL subfolder path - tsl_folder_path = os.path.join(script_directory, "TSL") - - # Create the TSL subfolder if it doesn't exist - if not os.path.exists(tsl_folder_path): - os.makedirs(tsl_folder_path) - - # Save data to a file in the TSL subfolder - filename = f"{country_code}.xml" - file_path = os.path.join(tsl_folder_path, filename) - - with open(file_path, 'w', encoding='utf-8') as file: - file.write(response_str) - - # Mark this URL as processed - processed_urls.add(tsl_url) - - def process_country(self, input_country, xmldoc, is_dev_build, processed_urls): - """Process XML data for a specific country and download TSL if applicable.""" - pointers_to_other_tsl = xmldoc.getElementsByTagName('PointersToOtherTSL') - - for pointer in pointers_to_other_tsl: - other_tsl_pointers = pointer.getElementsByTagName('OtherTSLPointer') - - for pointer_data in other_tsl_pointers: - tsl_locations = pointer_data.getElementsByTagName('TSLLocation') - additional_info = pointer_data.getElementsByTagName('AdditionalInformation') - - for info in additional_info: - other_info = info.getElementsByTagName('OtherInformation') - - for other_data in other_info: - scheme_territory = other_data.getElementsByTagName('SchemeTerritory') - mime_type = other_data.getElementsByTagName('ns3:MimeType') or other_data.getElementsByTagName('ns4:MimeType') - test_mime_type = other_data.getElementsByTagName('tslx:MimeType') - - for territory in scheme_territory: - pass - - if "_T" in input_country: - for test_type in test_mime_type: - if territory.firstChild.nodeValue == input_country and test_type.firstChild.nodeValue == 'application/vnd.etsi.tsl+xml': - tsl_url = pointer_data.getElementsByTagName('TSLLocation')[0].firstChild.nodeValue - self.download_tsl(tsl_url, territory.firstChild.nodeValue, is_dev_build, processed_urls) - break - else: - for mime in mime_type: - if territory.firstChild.nodeValue == input_country and mime.firstChild.nodeValue == 'application/vnd.etsi.tsl+xml': - self.mimetypeFound = True - tsl_url = pointer_data.getElementsByTagName('TSLLocation')[0].firstChild.nodeValue - self.download_tsl(tsl_url, territory.firstChild.nodeValue, is_dev_build, processed_urls) - break - - def main(self): - """Main function to execute the TSL downloader.""" - # Parse arguments - args = self.parse_arguments() - - input_countries = args.countries.split(';') if ';' in args.countries else [args.countries] - is_dev_build = args.isDevBuild.lower() == "true" - - # Processed TSL URLs - processed_urls = set() - - for input_country in input_countries: - xmldoc = self.load_xml(args.tslFile, args.tslTestFile, "_T" in input_country) - - if xmldoc is None: +TSL_MIME_TYPE = 'application/vnd.etsi.tsl+xml' +DOWNLOAD_TIMEOUT_SECONDS = 60 +DOWNLOAD_DIRECTORY = Path(__file__).resolve().parent / 'TSL' + + +def parse_arguments(): + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description='TSL XML Downloader') + parser.add_argument('--tslFile', action='append', default=[], type=Path, metavar='PATH', + help='Base file (LOTL) to look members up in. Repeatable, searched in order') + parser.add_argument('--members', required=True, type=str, + help='Scheme territories to download, e.g. "EE;EE_T;SE_TESTING"') + return parser.parse_args() + + +def child_elements(parent, local_name): + """Direct child elements with this local name, whatever XML namespace prefix the file uses.""" + return [ + node + for node in parent.childNodes + if node.nodeType == node.ELEMENT_NODE and node.localName == local_name + ] + + +def text_of(element): + """Text content of an element, or None when it is empty.""" + return element.firstChild.nodeValue if element.firstChild is not None else None + + +def other_information(pointer, local_name): + """Values of AdditionalInformation/OtherInformation/ for one pointer.""" + return [ + text_of(element) + for additional_information in child_elements(pointer, 'AdditionalInformation') + for information in child_elements(additional_information, 'OtherInformation') + for element in child_elements(information, local_name) + ] + + +def find_tsl_url(member, lotl): + """The TSL location the base file points at for this member, or None.""" + for pointers in lotl.getElementsByTagName('PointersToOtherTSL'): + for pointer in child_elements(pointers, 'OtherTSLPointer'): + if member not in other_information(pointer, 'SchemeTerritory'): + continue + if TSL_MIME_TYPE not in other_information(pointer, 'MimeType'): continue - - # Process country-specific data - self.process_country(input_country, xmldoc, is_dev_build, processed_urls) - # Validate country and mime type - if len(input_country) == 2 and not self.mimetypeFound: - raise Exception(f"Could not find mimetype for '{input_country}'. Verify input country code and check if the MimeType tag name has changed (nsx:MimeType) in TSL file.") + locations = child_elements(pointer, 'TSLLocation') + + if locations: + return text_of(locations[0]) + + return None + + +def existing_base_files(paths): + """The given base files that exist, in the order given, reporting the ones that do not.""" + existing = [] + + for path in paths: + if path.is_file(): + existing.append(path) + else: + print(f"Skipping missing base file '{path}'") + + return existing + + +def resolve_member(member, base_files): + """Look the member up in each base file in turn. Returns the first TSL location found.""" + for base_file in base_files: + tsl_url = find_tsl_url(member, minidom.parse(str(base_file))) + + if tsl_url is not None: + print(f"Found '{member}' in '{base_file}'") + return tsl_url + + print(f"No '{member}' pointer in '{base_file}'") + + return None + + +def download_tsl(tsl_url, destination_path): + """Download a TSL and save it.""" + print(f"Downloading {tsl_url} to {destination_path}") + + with urllib.request.urlopen(tsl_url, timeout=DOWNLOAD_TIMEOUT_SECONDS) as response: + destination_path.write_bytes(response.read()) + + +def main(): + """Main function to execute the TSL downloader.""" + args = parse_arguments() + members = [member for member in args.members.split(';') if member] + base_files = existing_base_files(args.tslFile) + + DOWNLOAD_DIRECTORY.mkdir(parents=True, exist_ok=True) + + for member in members: + tsl_url = resolve_member(member, base_files) + + if tsl_url is None: + searched = ', '.join(str(base_file) for base_file in base_files) or 'none' + raise SystemExit( + f"Could not find a TSL pointer for '{member}'. Base files searched: {searched}. " + f"Either the member code is wrong, or the base file listing it was not given." + ) + + download_tsl(tsl_url, DOWNLOAD_DIRECTORY / f"{member}.xml") if __name__ == "__main__": - downloader = TSLDownloader() - downloader.main() + main()