Files to database converter

util/sectors2sqlite.py

@@ -0,0 +1,135 @@
+#!/usr/bin/python3
+
+# Loads block files from sectors folders into map.sqlite database.
+# The sectors folder should be safe to remove after this prints "Finished."
+
+import time, os
+
+try:
+	import sqlite3
+except:
+	exit('You need to have the Python sqlite3 module.')
+
+path = "../world/"
+
+paths = []
+
+# sectors2 gets to try first
+if os.path.isdir(path + 'sectors2/'):
+	paths.append('sectors2')
+if os.path.isdir(path + 'sectors/'):
+	paths.append('sectors')
+
+if not paths:
+	exit('Could not find sectors folder at ' + path + 'sectors2/ or ' + path + 'sectors/')
+
+def uint(u):
+	u = int('0x'+u, 16)
+	return (u if u < 2**11 else u - 2**12)
+
+def int64(u):
+	while u >= 2**63:
+		u -= 2**64
+	while u <= -2**63:
+		u += 2**64
+	return u
+
+# Convert sector folder(s) to integer
+def getSectorPos(dirname):
+	if len(dirname) == 8:
+		# Old layout
+		x = uint(dirname[:4])
+		z = uint(dirname[4:])
+	elif len(dirname) == 7:
+		# New layout
+		x = uint(dirname[:3])
+		z = uint(dirname[4:])
+	else:
+		print('Terrible sector at ' + dirname)
+		return
+	
+	return x, z
+
+# Convert block file to integer position
+def getBlockPos(sectordir, blockfile):
+	p2d = getSectorPos(sectordir)
+	
+	if not p2d:
+		return
+	
+	if len(blockfile) != 4:
+		print("Invalid block filename: " + blockfile)
+	
+	y = uint(blockfile)
+	
+	return p2d[0], y, p2d[1]
+
+# Convert location to integer
+def getBlockAsInteger(p):
+	return int64(p[2]*16777216 + p[1]*4096 + p[0])
+
+# Init
+
+create = False
+if not os.path.isfile(path + 'map.sqlite'):
+	create = True
+
+conn = sqlite3.connect(path + 'map.sqlite')
+
+if not conn:
+	exit('Could not open database.')
+
+cur = conn.cursor()
+
+if create:
+	cur.execute("CREATE TABLE IF NOT EXISTS `blocks` (`pos` INT NOT NULL PRIMARY KEY, `data` BLOB);")
+	conn.commit()
+	print('Created database at ' + path + 'map.sqlite')
+
+# Crawl the folders
+
+count = 0
+t = time.time()
+for base in paths:
+	v = 0
+	if base == 'sectors':
+		v = 1
+	elif base == 'sectors2':
+		v= 2
+	else:
+		print('Ignoring base ' + base)
+		continue
+	
+	for root, dirs, files in os.walk(path + base):
+		if files:
+			for block in files:
+				pos = getBlockAsInteger(getBlockPos(root[(-8 if v == 1 else -7 if v == 2 else 0):], block))
+				
+				if pos is None:
+					print('Ignoring broken path ' + root + '/' + block)
+					continue
+				
+				f = open(root+'/'+block, 'rb')
+				cur.execute('INSERT OR IGNORE INTO `blocks` VALUES(?, ?)', (pos, f.read()))
+				f.close()
+				count += 1
+		
+		if(time.time() - t > 3):
+			t = time.time()
+			print(str(count)+' blocks processed...')
+
+conn.commit()
+
+print('Finished. (' + str(count) + ' blocks)')
+
+
+
+
+
+
+
+
+
+
+
+