diff --git a/README.md b/README.md index c7d2e15..68ee67d 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,24 @@ # python-mbtiles2compactcache (credits: https://github.com/Esri/raster-tiles-compactcache) -## Compact Cache V2 sample code +## Compact Cache V2 ### mbtiles2compactcache.py -Convert individual .mbtile files to the [Esri Compact Cache V2](./CompactCacheV2.md) format bundles. It only builds individual bundles, not a completely functional cache. +Convert a single raster/vector tiles .mbtile file to the [Esri Compact Cache V2](./CompactCacheV2.md) format bundles. It only builds a completely functional cache. This script is designed to export large to huge raster dataset mbtiles. the export occurs using multiple threads reading all records sequentially. -While operational, this code is only provided as an example of how a bundle file is created and updated. -This Python script takes two arguments, the input mbtile folder and the output folder. It assumes that the input folder contains mbtile files of the form: ```\.mbtile```. +Requirement: +- data must be in Web Mercator (EPSG:3857) +- the tiles table must be a rowid table. (if the tiles table is a view, please add the rowid field to the view.) The script does not check the input tile format, and assumes that all the files under the source contain valid SQLLite databases with tiles in MBTiles format. -The algorithm loops over files, inserting each tile in the appropriate bundle. It keeps one bundle open in case the next tile fits in the same bundle. In most cases this combination results in good performance. +The algorithm loops over the records, inserting each tile in the appropriate bundle. Each thread writes its records in a bundle and then close it. -The [sample_mbtiles](./sample_mbtiles) folder contains example [MBTiles](./sample_mbtiles/README.md) the form of SQLite databases for the single zoom levels for the first three level of the Federal Agency for Cartography and Geodesy - TopPlusOpen cache in Web Mercator projection. The [sample_cache] (../sample_cache) folder contains a Compact Cache V2 cache produced from these individual mbtiles using the mbtiles2compactcache.py script. The commands used to generate the bundles for each level are: +The [file](./file) folder contains example [MBTiles] +The [cache] (./cache) folder contains a Compact Cache V2 cache produced as result of the mbtiles2compactcache.py script. The commands used to generate the cache is: -RGB processing: ```console -python sample_code\mbtiles2compactcache.py -i sample_mbtiles -o sample_cache\_alllayers -``` -Grayscale processing: -```console -python sample_code\mbtiles2compactcache.py -i sample_mbtiles -o sample_cache\_alllayers -g +python .\code\mbtiles2compactcache.py -ml 15 -s .\file\countries-raster.mbtiles -d .\cache\A3_MyCachedService\Layers\_alllayers ``` ## Documentation and sample code for Esri Compact Cache V2 format @@ -34,7 +31,7 @@ The Compact Cache V2 is the current file format used by ArcGIS to store raster t | Row 1 | Row 1 Col 0 | Row 1 Col 1 | ## Content -This repository contains [documentation](CompactCacheV2.md), a [sample cache](sample_cache) and a Python 2.x [code example](sample_code) of how to build Compact Cache V2 bundles from MBTiles. +This repository contains [documentation](CompactCacheV2.md), a [cache](cache) and a Python 3.x [code example](code) of how to build Compact Cache V2 bundles from MBTiles. ## Licensing diff --git a/code/check_contiguous_tiles.py b/code/check_contiguous_tiles.py new file mode 100644 index 0000000..1ad29a0 --- /dev/null +++ b/code/check_contiguous_tiles.py @@ -0,0 +1,101 @@ +# ------------------------------------------------------------------------------- +# Name: check_contiguous_tiles +# Purpose: Check if there is no hole (missing tiles) in a bundle. +# +# Author: ltbam +# +# Created: 18/07/2022 +# Modified: - +# +# Copyright 2023 swisstopo. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.? +# +# ------------------------------------------------------------------------------- +# Changeset +# Version 1.0.0 ltbam +from mbtilesRaster2compactcache import Bundle +import os +import math + + +cache_output_folder = r"C:\Temp\osm\A3_MyCachedService\Layers" + +def main(): + print("Checking contiguous tiles in Bundle") + for path, subdirs, files in os.walk(cache_output_folder): + for name in files: + if name.endswith(".bundle"): + bdl = Bundle(os.path.join(path, name)) + bdl.open() + results = listMissingTiles(bdl) + if len(results) == 0: + print("bundle {} is ok".format(name)) + else: + for res in results: + print("Missing contiguous tile: level {}, row {}, col {}".format(res["lvl"], res["row"], + res["col"])) + # close bundle without writing anything + bdl.fd.close() + bdl.fd = None + +def listMissingTiles(bundle): + files = [] + # Loop each Tile index and resolve if it has data + # range(0, 128) means 0-127 + for row in range(0, 128): + startTile = 0 + numTiles = 0 + # count tiles with data, determine drawing center + for col in range(0, 128): + t_idx = bundle.curr_index[128 * row + col] + t_size = int(math.floor(t_idx / Bundle.M)) + if t_size > 0: + numTiles += 1 + if startTile == 0: + startTile = col + + if numTiles > 3: + data_started = False + mid_range = startTile + numTiles // 2 + # print("lvl {} row {} mid_range: {}".format(bundle.level, row, mid_range)) + # inspect from left + # range(0, 128) means 0-127 + for col in range(0, mid_range): + t_idx = bundle.curr_index[128 * row + col] + t_size = int(math.floor(t_idx / Bundle.M)) + if data_started: + if t_size == 0: + absrow = bundle.row_offset + row + abscol = bundle.col_offset + col + files.append(dict(col=abscol, row=absrow, lvl=int(bundle.level))) + else: + if t_size != 0: + data_started = True + data_started = False + # inspect from right + for col in range(127, mid_range - 1, -1): + t_idx = bundle.curr_index[128 * row + col] + t_size = int(math.floor(t_idx / Bundle.M)) + if data_started: + if t_size == 0: + absrow = bundle.row_offset + row + abscol = bundle.col_offset + col + files.append(dict(col=abscol, row=absrow, lvl=int(bundle.level))) + else: + if t_size != 0: + data_started = True + + return files + +if __name__ == '__main__': + main() diff --git a/code/mbtiles2compactcache.py b/code/mbtiles2compactcache.py new file mode 100644 index 0000000..d6f1bb8 --- /dev/null +++ b/code/mbtiles2compactcache.py @@ -0,0 +1,434 @@ +# ------------------------------------------------------------------------------- +# Name: mbtilesRaster2compactcache +# Purpose: Build compact cache V2 bundles from single MBTiles Raster dataset file. +# +# Author: ltbam, luci6974 +# +# Created: 18/07/2022 +# Modified: - +# +# Copyright 2022 swisstopo. 2016 ESRI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.? +# +# ------------------------------------------------------------------------------- +# +# Converts .mbtile file to the esri Compact Cache V2 format +# +# Takes two arguments, the first one is the input .mbfile folder +# the second one being the output cache folder (_alllayers) +# +# +# This script is intended to transform big Files, so it will loop each record +# and resolve is it has to be exported based on the maxlvl parameter. +# +# It does not check the input tile format, and assumes that the file +# is a valid sqlite tile databases. +# +# ------------------------------------------------------------------------------- +# +# Changeset +# Version 1.0.0 ltbam + +import argparse +import sqlite3 +import os +import struct +import shutil +import datetime +import re +import io +import time +import math + +from queue import Queue +from threading import Lock, get_ident, Thread +import multiprocessing + + +class Bundle: + # Bundle linear size in tiles + BSZ = 128 + # Tiles per bundle + BSZ2 = BSZ ** 2 + # Index size in bytes + IDXSZ = BSZ2 * 8 + # Max size + M = 2 ** 40 + + def __init__(self, file_name): + self.file_name = file_name + self.curr_max = 0 + self.curr_offset = 0 + self.curr_index = [] + self.lock = Lock() + self.fd = None + regex = r"L(..)" + self.level = re.findall(regex, self.file_name)[0] + fname = self.file_name.split('.bundle')[0].split('\\')[-1] + s_val = fname[1:].split('C') + self.row_offset = int(s_val[0], 16) + self.col_offset = int(s_val[1], 16) + if not os.path.exists(self.file_name): + self.init_content() + + def init_content(self): + # print("t {0}: initializing: {1}".format(get_ident(), self.file_name)) + self.fd = open(self.file_name, "wb") + # Empty bundle file header, lots of magic numbers + header = struct.pack("<4I3Q6I", + 3, # Version + Bundle.BSZ2, # numRecords + 0, # maxRecord Size + 5, # Offset Size + 0, # Slack Space + 64 + Bundle.IDXSZ, # File Size + 40, # User Header Offset. + 20 + Bundle.IDXSZ, # User Header Size + 3, # Legacy 1 + 16, # Legacy 2 + Bundle.BSZ2, # Legacy 3 + 5, # Legacy 4 + Bundle.IDXSZ # Index Size + ) + self.fd.write(header) + # Write empty index. + self.fd.write(struct.pack("<{}Q".format(Bundle.BSZ2), *((0,) * Bundle.BSZ2))) + self.fd.close() + self.fd = None + + def open(self): + # Open the bundle + # wait a bit if a thread closed a File before opening it again + self.fd = open(self.file_name, "r+b") + # Read the current max record size + self.fd.seek(8) + self.curr_max = int(struct.unpack(" - - - - PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0],AUTHORITY["EPSG",3857]] - -20037700 - -30241100 - 148923141.92838538 - -100000 - 10000 - -100000 - 10000 - 0.001 - 0.001 - 0.001 - true - 102100 - 3857 - - - -20037508.342787001 - 20037508.342787001 - - 256 - 256 - 96 - - - 0 - 591657527.591555 - 156543.03392799999 - - - 1 - 295828763.79577702 - 78271.516963999893 - - - 2 - 147914381.89788899 - 39135.758482000099 - - - 3 - 73957190.948944002 - 19567.879240999901 - - - 4 - 36978595.474472001 - 9783.9396204999593 - - - 5 - 18489297.737236001 - 4891.9698102499797 - - - 6 - 9244648.8686180003 - 2445.9849051249898 - - - 7 - 4622324.4343090001 - 1222.9924525624899 - - - 8 - 2311162.2171550002 - 611.49622628138002 - - - 9 - 1155581.108577 - 305.74811314055802 - - - 10 - 577790.55428899999 - 152.874056570411 - - - 11 - 288895.27714399999 - 76.437028285073197 - - - 12 - 144447.638572 - 38.218514142536598 - - - 13 - 72223.819285999998 - 19.109257071268299 - - - 14 - 36111.909642999999 - 9.5546285356341496 - - - 15 - 18055.954822 - 4.7773142679493699 - - - 16 - 9027.9774109999998 - 2.38865713397468 - - - 17 - 4513.9887049999998 - 1.1943285668550501 - - - 18 - 2256.994353 - 0.59716428355981699 - - - 19 - 1128.4971760000001 - 0.29858214164761698 - - - 96 - - - PNG32 - 0 - false - 1 - 0 - - - esriMapCacheStorageModeCompactV2 - 128 - - \ No newline at end of file diff --git a/sample_code/README.md b/sample_code/README.md deleted file mode 100644 index 9789163..0000000 --- a/sample_code/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Compact Cache V2 sample code - -## mbtiles2compactcache.py - -Convert individual .mbtile files to the [Esri Compact Cache V2](../CompactCacheV2.md) format bundles. It only builds individual bundles, not a completely functional cache. - -While operational, this code is only provided as an example of how a bundle file is created and updated. -This Python script takes two arguments, the input mbtile folder and the output folder. It assumes that the input folder contains mbtile files of the form: ```\.mbtile```. - -The script does not check the input tile format, and assumes that all the files under the source contain valid SQLLite databases with tiles in MBTiles format. -The algorithm loops over files, inserting each tile in the appropriate bundle. It keeps one bundle open in case the next tile fits in the same bundle. In most cases this combination results in good performance. - -The [sample_mbtiles](../sample_mbtiles) folder contains example [MBTiles](../sample_mbtiles/README.md) the form of SQLite databases for the single zoom levels for the first three level of the Federal Agency for Cartography and Geodesy - TopPlusOpen cache in Web Mercator projection. The [sample_cache] (../sample_cache) folder contains a Compact Cache V2 cache produced from these individual mbtiles using the mbtiles2compactcache.py script. The commands used to generate the bundles for each level are: - -RGB processing: -``` console -python sample_code\mbtiles2compactcache.py -i sample_mbtiles -o sample_cache\_alllayers -``` -Grayscale processing: -``` console -python sample_code\mbtiles2compactcache.py -i sample_mbtiles -o sample_cache\_alllayers -g -``` - -## Licensing - -Copyright 2018 Esri Deutschland GmbH - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - -A copy of the license is available in the repository's license.txt file. - diff --git a/sample_code/mbtiles2compactcache.py b/sample_code/mbtiles2compactcache.py deleted file mode 100644 index 1ea27b7..0000000 --- a/sample_code/mbtiles2compactcache.py +++ /dev/null @@ -1,404 +0,0 @@ -# ------------------------------------------------------------------------------- -# Name: mbtiles2compactcache -# Purpose: Build compact cache V2 bundles from MBTiles in SQLLite databases -# -# Author: luci6974 -# -# Created: 20/09/2016 -# Modified: 04/05/2018,esristeinicke -# 23/10/2019,mimo -# -# Copyright 2016 Esri -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License.? -# -# ------------------------------------------------------------------------------- -# -# Converts .mbtile files to the esri Compact Cache V2 format -# -# Takes two arguments, the first one is the input .mbfile folder -# the second one being the output cache folder (_alllayers) -# -# -# Assumes that the input .mbtile files are named after the level.. (17.mbtile) -# -# Loops over columns and then row, in the order given by os.walk -# Keeps one bundle open in case the next tile fits in the same bundle -# In most cases this combination results in good performance -# -# It does not check the input tile format, and assumes that all -# the files are valid sqlite tile databases. In other -# words, make sure there are no spurious files and folders under the input -# path, otherwise the output bundles might have strange content. -# -# ------------------------------------------------------------------------------- -# -# v1.2 added grayscale Option (requires pillow) -# * to install pillow: -# * make sure python & scripts/pip is in path -# * in cmd type pip install pillow -# -# v1.3 fixed grayscale (Grayscale + Alpha = fixed grayscale Image) & (96 DPI) -# -# v1.4 better logging / fixing for python 3 / fixed ETA -# -# v1.5 better parameter support & parameter help -import argparse -import sqlite3 -import os -import struct -import shutil -import datetime -import re -import io - -try: - from PIL import Image - is_pillow = True -except ImportError as import_error: - is_pillow = False - - - -# Bundle linear size in tiles -BSZ = 128 -# Tiles per bundle -BSZ2 = BSZ ** 2 -# Index size in bytes -IDXSZ = BSZ2 * 8 - -# Output path -output_path = None - -# The curr_* variable are used for caching of open output bundles -# current bundle is kept open to reduce overhead -# TODO: Eliminate global variables -curr_bundle = None -# A bundle index list -# Array would be better, but it lacks 8 byte int support -curr_index = None -# Bundle file name without path or extension -curr_bname = None -# Current size of bundle file -# curr_offset = long(0) -curr_offset = int(0) -# max size of a tile in the current bundle -curr_max = 0 - - -def get_arguments(): - """ - Parses commandline arguments. - - :return: commandline arguments - """ - parser = argparse.ArgumentParser() - - parser.add_argument('-s', '--source', - help='Input folder containing the mbtile files.', required=True) - parser.add_argument('-d', '--destination', - help='Output for level folders.', required=True) - parser.add_argument('-l', '--level', - help='Do only this Level (useful for parallel starts with Grayscale).', default=-1, type=int, - required=False) - parser.add_argument('-g', '--grayscale', - help='Convert tiles to grayscale while processing.', default=False, action="store_true", - required=False) - - # Return the command line arguments. - arguments = parser.parse_args() - - # validate folder parameters - if not is_pillow and arguments.grayscale: - parser.error("Grayscale option requires Pillow (PIL) module to be installed.") - if not os.path.exists(arguments.source): - parser.error("Input folder does not exist or is inaccessible.") - if not os.path.exists(arguments.destination): - parser.error("Output folder does not exist or is inaccessible.") - - return arguments - - -def init_bundle(file_name): - """Create an empty V2 bundle file - :param file_name: bundle file name - """ - fd = open(file_name, "wb") - # Empty bundle file header, lots of magic numbers - header = struct.pack("<4I3Q6I", - 3, # Version - BSZ2, # numRecords - 0, # maxRecord Size - 5, # Offset Size - 0, # Slack Space - 64 + IDXSZ, # File Size - 40, # User Header Offset - 20 + IDXSZ, # User Header Size - 3, # Legacy 1 - 16, # Legacy 2 - BSZ2, # Legacy 3 - 5, # Legacy 4 - IDXSZ # Index Size - ) - fd.write(header) - # Write empty index. - fd.write(struct.pack("<{}Q".format(BSZ2), *((0,) * BSZ2))) - fd.close() - - -def cleanup(): - """ - Updates header and closes the current bundle - """ - global curr_bundle, curr_bname, curr_index, curr_max, curr_offset - curr_bname = None - - # Update the max rec size and file size, then close the file - if curr_bundle is not None: - curr_bundle.seek(8) - curr_bundle.write(struct.pack("PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0],AUTHORITY["EPSG",3857]]-20037700-30241100148923141.92838538-10000010000-100000100000.0010.0010.001true1021003857-20037508.34278700120037508.342787001256256960591657527.591555156543.033927999991295828763.7957770278271.5169639998932147914381.8978889939135.758482000099373957190.94894400219567.879240999901436978595.4744720019783.9396204999593518489297.7372360014891.969810249979769244648.86861800032445.984905124989874622324.43430900011222.992452562489982311162.2171550002611.4962262813800291155581.108577305.7481131405580210577790.55428899999152.87405657041111288895.2771439999976.43702828507319712144447.63857238.2185141425365981372223.81928599999819.1092570712682991436111.9096429999999.55462853563414961518055.9548224.777314267949369996PNG320false10esriMapCacheStorageModeCompactV2128 \ No newline at end of file