diff options
Diffstat (limited to 'devtools')
29 files changed, 3820 insertions, 0 deletions
diff --git a/devtools/blade_runner/subtitles/.gitignore b/devtools/blade_runner/subtitles/.gitignore new file mode 100644 index 0000000000..894a44cc06 --- /dev/null +++ b/devtools/blade_runner/subtitles/.gitignore @@ -0,0 +1,104 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/devtools/blade_runner/subtitles/README.md b/devtools/blade_runner/subtitles/README.md new file mode 100644 index 0000000000..12df03c170 --- /dev/null +++ b/devtools/blade_runner/subtitles/README.md @@ -0,0 +1,108 @@ +# Blade Runner Subtitles + +# Blade Runner (1997) Subtitles Support +Some tools written in Python 2.7 to help add support for subtitles in Westwood's point and click adventure game Blade Runner (1997) for PC. + +## quotesSpreadsheetCreator (sortBladeRunnerWavs##) +(requires python lib *xlwt*, *wave*) +A tool to gather all the speech audio filenames in an Excel file which will include a column with links to the audio file location on the PC. By Ctrl+MouseClick on that column's entries you should be able to listen to the corresponding wav file. +The output excel file *out.xls* should help with the transcription of all the spoken *in-game* quotes. It also provides extra quote information such as the corresponding actor ID and quote ID per quote. + +Note 1: A lot of extra information has been added to the output Excel file maintained for the English transcription, such as whether a quote is unused or untriggered, the person a quote refers to (when applicable), as well as extra quotes that are not separate Audio files (AUD) in the game's archives but are part of a video file (VQA) or were text resources (TRE) for dialogue menus, UI etc. Therefore, this tool is provided here mostly for archiving purposes. +__The online Excel file is available here:__ +https://docs.google.com/spreadsheets/d/17ew0YyhSwqcqZg6bXrIgz0GkA62dhgViHN15lOu5Hj8/edit?usp=sharing + +Note 2: Using the "-xwav" switch, this tool will export __ALL__ game's audio files (that are either speech or speech-related) in a WAV format. This is expected to take up quite a lot of your HDD space. + +Usage: +``` +python2.7 sortBladeRunnerWavs02.py -ip <folderpath_for_TLK_Files> -op <folderpath_for_extracted_wav_Files> -m <stringPathToReplaceFolderpathInExcelLinks> [-xwav] [-xtre] +``` +The tool __requires__ the actornames.txt file, which is included in the samples folder, to be in the same folder as the tool's source (.py) file. + + +## mixResourceCreator (packBladeRunnerMIXFromPCTLKXLS-##) +(requires python lib *xlrd*) +A tool to process the aforementioned Excel file with the dialogue transcriptions and output text resource files (TRE) that will be packed along with the external font (see fontCreator tool) into a SUBTITLES.MIX file. Currently, a modified version of the ScummVM's BladeRunner engine is required for this MIX file to work in-game. Multiple TRE files will be created intermediately in order to fully support subtitles in the game. One TRE file includes all in-game spoken quotes and the rest of them correspond to any VQA video sequence that contain voice acting. +Usage: +``` +python2.7 packBladeRunnerMIXFromPCTLKXLS-04.py -x <excelWithTranscriptSheets.xls> +``` +The tool __requires__ the actornames.txt file, which is included in the samples folder, to be in the same folder as the tool's source (.py) file. + +## fontCreator (grabberFromPNG##BR) +(requires python image library *PIL*) +A tool to support __both__ the extraction of fonts from the game __and__ the creation of a font file (FON) for use with (currently) a modified version of ScummVM's BladeRunner engine (WIP) in order to resolve various issues with the available fonts (included in the game's own resource files). These issues include alignment, kerning, corrupted format, limited charset and unsupported characters -- especially for languages with too many non-Latin symbols in their alphabet. +This font tool's code is based off the Monkey Island Special Edition's Translator (https://github.com/ShadowNate/MISETranslator). +Usage: +``` +Syntax A - To export game fonts: +python2.7 grabberFromPNG17BR.py -ip <folderpathForMIXFiles> + +Syntax B - To create subtitle font: +python2.7 grabberFromPNG17BR.py -im <imageRowPNGFilename> -om <targetFONfilename> -pxLL <minSpaceBetweenLettersInRowLeftToLeft> -pxTT <minSpaceBetweenLettersInColumnTopToTop> -pxKn <kerningForFirstDummyFontLetter> -pxWS <whiteSpaceWidthInPixels> +``` +This tool also __requires__ an overrideEncoding.txt file to be in the same folder as the tool's source (.py) file. +The overrideEncoding.txt is a __text file that should be saved in a UTF-8 encoding (no BOM)__, that contains the following: +1. A key "targetEncoding" with a value of the name of the ASCII codepage that should be used for the character fonts (eg windows-1253). +2. A key "asciiCharList" with value the "all-characters" string with all the printable characters that will be used in-game, from the specified codepage. Keep in mind that: + * The first such character (typically this is the '!' character) should be repeated twice! + * All characters must belong to the specified codepage. + * The order that the characters appear in the string should match their order in the ASCII table of the codepage. + * You don't need to use all the characters of the specified codepage in your "all-characters" string. + * For any special characters that don't appear in the target codepage (eg ñ, é, í, â don't appear in the Greek codepage), you'll have to decide on an ASCII value for them (one not used by another character appearing in-game). + In the "all-characters" string you should put as placeholders the actual characters from the specified codepage that correspond to the ASCII values you have decided above; The placeholder characters should also be in the proper order (by their ASCII value) inside the string. +3. A key "explicitKerningList" with value a list of comma separated tuples where each tuple specifies a character and its manually set kerning (x-offset) in pixels. Kerning can have integer (positive or negative) values. This list is optional and may be skipped, if you put a '-' instead of a list. + * Example: explicitKerningList=i:-1 + * Don't use space(s) between the tuples! +4. A key "explicitWidthIncrement" with value a list of comma separated tuples where each tuple specifies a character and its manually set extended width in pixels. This should be a positive integer value. You can skip this list by not writing anything in the file after the previous (manual kerning) list. + * Example: explicitWidthIncrement=i:0,j:1,l:1 + * Don't use space(s) between the tuples! +5. A key "originalFontName" with the FON file's original name in the game (the one that it should replace). Use SUBLTS for the subtitles FON. + * Example: originalFontName=SUBLTS +6. A key "specialOutOfOrderGlyphsUTF8ToAsciiTargetEncoding" with value a list of comma separated tuples that indicates which character glyphs should replace the placeholder glyphs in your all-character string above. + * Example: specialOutOfOrderGlyphsUTF8ToAsciiTargetEncoding=í:Ά,ñ:¥,â:¦,é:§,Ά:£ + * Don't use space(s) between the tuples! +There is a sample of such file in the source folder for the fontCreator tool. + + +__For the exporting the game fonts mode__, the valid syntax expects only one (1) argument: +1. folderpathForMIXFiles: is the path where the game's MIX files are located (STARTUP.MIX is required). The exported font files will be: 10PT.FON, TAHOMA18.FON, TAHOMA24.FON and KIA6PT.FON. + +__For the creation of subtitles' font mode__, there are six (6) mandatory launch arguments for the fontCreator tool: +1. imageRowPNGFilename: is the filename of the input PNG image file which should contain a row of (preferably) tab separated glyphs. Example: "Tahoma_18ShdwTranspThreshZero003-G5.png". Keep in mind that: + * The first glyph should be repeated here too, as in the overrideEncoding.txt file. + * Background should be transparent. + * All colors used in the character glyphs should not have any transparency value (eg from Gimp 2, set Layer->Transparency->Threshold alpha to 0). There's no partial transparency supported by the game. A pixel will either by fully transparent or fully opaque. + * If you use special glyphs that are not in the specified ASCII codepage (eg ñ, é, í, â don't appear in the Greek codepage), then in this image file you should use the actual special glyphs - put them at the position of the placeholder characters in your "all-characters" string that you've specified in the overrideEncoding.txt file. +2. targetFONfilename: Example: "SUBTLS_E.FON". Keep in mind that: + * As of yet, only the SUBTLS_E.FON is supported by a modified (non-official) version of the BladeRunner ScummVM engine. +3. minSpaceBetweenLettersInRowLeftToLeft: This is a length (positive integer) in pixels that indicates the __minimum__ distance between the left-most side of a glyph and the left-most side of the immediate subsequent glyph in the input image PNG (row of glyphs) file. +This basically tells the tool how far (in the x axis) it can search for pixels that belong to the same glyph). You can input an approximate value here and adjust it based on the output of the tool (the tool should be able to detect ALL the glyphs in the PNG row image file and it will report how many it detected in its output) +4. minSpaceBetweenLettersInColumnTopToTop: This is a positive integer in pixels that indicates the __minimum__ distance between the top-most pixel of a glyph and the top-most pixel of a glyph on another row of the input image file. +It is highly recommended, though, that the input image file should contain only a single row of glyphs and this value should be higher than the maximum height among the glyphs, typically this should be set to approximately double the maximum height of glyph. +5. kerningForFirstDummyFontGlyph: This is an integer that explicitly indicates the kerning, ie. offset in pixels (on the x-axis) of the first glyph (the one that is repeated twice). This can be measured by observing the indent that your image processing app adds when you enter the first glyph (typically it should be only a few pixels) +6. whiteSpaceWidthInPixels: This is a positive integer value that sets the width in pixels for the single white space between words for the subtitles in-game. + +A suggested method of creating decent looking PNG with the row of glyphs for your subtitles' font is the following: +1. Create the font row in __GIMP__ + * Start with a __new__ empty image, with transparent background. Choose a large enough canvas width (you can adjust it later) + * Paste as a new layer a tab separated alphanumeric sequence with all your glyphs (as specified above). Choose white as the font's color. + * Adjust your canvas' width and height to make sure all the glyphs are within its borders. +2. Add layers for shadows if necessary (recommended) by duplicating the original layer with the (white colored) glyphs to create layers that would be used for the shadow effect. Those layers should be __colorified__ as black and placed behind the original layer, displaced by one (1) pixel from eg. the top, right, left, and bottom (it's recommended to do this for all of those four). +3. __Merge all visible__ layers (maintaining an alpha channel for the background) +4. __Select all__ and __float the selection__, which should create a floating selection with all the letter glyphs. + * __Promote that selection to a layer__ and __duplicate__ it. +5. Choose one of the duplicated layers and __COLORIFY__ it to pitch black. + * __Set the transparency threshold__ of this black layer to 0. +6. Finally, place this completely black colored layer underneath the other one and __merge the visible__ layers. +7. Export your image to a PNG file. + +This should get rid of semi-transparent pixels while maintaining the "aliasing" effect. +There could be a better way but this should work ok. + + +# Credits and Special Thanks +- All the developer guys from the ScummVM (https://github.com/scummvm/scummvm) team, and especially the ones involved in the implementation of the BladeRunner engine for ScummVM (madmoose, peterkohaut, sev and everyone else). +- The information provided in this blog (http://westwoodbladerunner.blogspot.ca) by Michael Liebscher. +- The creator of br-mixer (https://github.com/bdamer/br-mixer), Ben Damer, who also has a blog entry about the game resource file formats (http://afqa123.com/2015/03/07/deciphering-blade-runner/) diff --git a/devtools/blade_runner/subtitles/fontCreator/fonFileLib.py b/devtools/blade_runner/subtitles/fontCreator/fonFileLib.py new file mode 100644 index 0000000000..c523162950 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/fonFileLib.py @@ -0,0 +1,233 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +# +import os, sys, shutil +import struct +from struct import * +import Image + +my_module_version = "0.50" +my_module_name = "fonFileLib" + + +class FonHeader: + maxEntriesInTableOfDetails = -1 # this is probably always the number of entries in table of details, but it won't work for the corrupted TAHOMA18.FON file + maxGlyphWidth = -1 # in pixels + maxGlyphHeight = -1 # in pixels + graphicSegmentByteSize = -1 # Graphic segment byte size + + def __init__(self): + return + + +class fonFile: + m_header = FonHeader() + + simpleFontFileName = "GENERIC.FON" + realNumOfCharactersInImageSegment = 0 # this is used for the workaround for the corrupted TAHOME18.FON + nonEmptyCharacters = 0 + + glyphDetailEntriesLst = [] # list of 5-value tuples. Tuple values are (X-offset, Y-offset, Width, Height, Offset in Graphics segment) + glyphPixelData = None # buffer of pixel data for glyphs + + def __init__(self): + del self.glyphDetailEntriesLst[:] + self.glyphPixelData = None # buffer of pixel data for glyphs + self.simpleFontFileName = "GENERIC.FON" + self.realNumOfCharactersInImageSegment = 0 # this is used for the workaround for the corrupted TAHOME18.FON + self.nonEmptyCharacters = 0 + + return + + def loadFonFile(self, fonBytesBuff, maxLength, fonFileName): + self.simpleFontFileName = fonFileName + + + offsInFonFile = 0 + localLstOfDataOffsets = [] + del localLstOfDataOffsets[:] + # + # parse FON file fields for header + # + try: + tmpTuple = struct.unpack_from('I', fonBytesBuff, offsInFonFile) # unsigned integer 4 bytes + self.header().maxEntriesInTableOfDetails = tmpTuple[0] + offsInFonFile += 4 + + if self.simpleFontFileName == 'TAHOMA18.FON': # deal with corrupted original 'TAHOMA18.FON' file + self.realNumOfCharactersInImageSegment = 176 + print "SPECIAL CASE. WORKAROUND FOR CORRUPTED %s FILE. Only %d characters supported!" % (self.simpleFontFileName, self.realNumOfCharactersInImageSegment) + else: + self.realNumOfCharactersInImageSegment = self.header().maxEntriesInTableOfDetails + + tmpTuple = struct.unpack_from('I', fonBytesBuff, offsInFonFile) # unsigned integer 4 bytes + self.header().maxGlyphWidth = tmpTuple[0] + offsInFonFile += 4 + + tmpTuple = struct.unpack_from('I', fonBytesBuff, offsInFonFile) # unsigned integer 4 bytes + self.header().maxGlyphHeight = tmpTuple[0] + offsInFonFile += 4 + + tmpTuple = struct.unpack_from('I', fonBytesBuff, offsInFonFile) # unsigned integer 4 bytes + self.header().graphicSegmentByteSize = tmpTuple[0] + offsInFonFile += 4 + + print "FON Header Info: " + print "Num of entries: %d\tGlyph max-Width: %d\tGlyph max-Height: %d\tGraphic Segment size %d" % (self.header().maxEntriesInTableOfDetails, self.header().maxGlyphWidth, self.header().maxGlyphHeight, self.header().graphicSegmentByteSize) + #print "Num of entries: %d\tGlyph max-Width: %d\tGlyph max-Height: %d\tGraphic Segment size %d" % (self.realNumOfCharactersInImageSegment, self.header().maxGlyphWidth, self.header().maxGlyphHeight, self.header().graphicSegmentByteSize) + # + # Glyph details table (each entry is 5 unsigned integers == 5*4 = 20 bytes) + # For most characters, their ASCII value + 1 is the index of their glyph's entry in the details table. The 0 entry of this table is reserved + # + #tmpXOffset, tmpYOffset, tmpWidth, tmpHeight, tmpDataOffset + print "FON glyph details table: " + for idx in range(0, self.realNumOfCharactersInImageSegment): + tmpTuple = struct.unpack_from('i', fonBytesBuff, offsInFonFile) # unsigned integer 4 bytes + tmpXOffset = tmpTuple[0] + offsInFonFile += 4 + + tmpTuple = struct.unpack_from('I', fonBytesBuff, offsInFonFile) # unsigned integer 4 bytes + tmpYOffset = tmpTuple[0] + offsInFonFile += 4 + + tmpTuple = struct.unpack_from('I', fonBytesBuff, offsInFonFile) # unsigned integer 4 bytes + tmpWidth = tmpTuple[0] + offsInFonFile += 4 + + tmpTuple = struct.unpack_from('I', fonBytesBuff, offsInFonFile) # unsigned integer 4 bytes + tmpHeight = tmpTuple[0] + offsInFonFile += 4 + + tmpTuple = struct.unpack_from('I', fonBytesBuff, offsInFonFile) # unsigned integer 4 bytes + tmpDataOffset = tmpTuple[0] + offsInFonFile += 4 + + if tmpWidth == 0 or tmpHeight == 0: + print "Index: %d\t UNUSED *****************************************************************" % (idx) + else: + self.nonEmptyCharacters += 1 + print "Index: %d\txOffs: %d\tyOffs: %d\twidth: %d\theight: %d\tdataOffs: %d" % (idx, tmpXOffset, tmpYOffset, tmpWidth, tmpHeight, tmpDataOffset) + if tmpDataOffset not in localLstOfDataOffsets: + localLstOfDataOffsets.append(tmpDataOffset) + else: + # This never happens in the original files. Offsets are "re-used" but not really because it happens only for empty (height = 0) characters which all seem to point to the next non-empty character + print "Index: %d\t RE-USING ANOTHER GLYPH *****************************************************************" % (idx) + + self.glyphDetailEntriesLst.append( ( tmpXOffset, tmpYOffset, tmpWidth, tmpHeight, tmpDataOffset) ) + + offsInFonFile = (4 * 4) + (self.header().maxEntriesInTableOfDetails * 5 * 4) # we need the total self.header().maxEntriesInTableOfDetails here and not self.realNumOfCharactersInImageSegment + self.glyphPixelData = fonBytesBuff[offsInFonFile:] + return True + except: + print "Loading failure!" + raise + return False + + def outputFonToPNG(self): + targWidth = 0 + targHeight = 0 + paddingFromTopY = 2 + paddingBetweenGlyphsX = 10 + + + if len(self.glyphDetailEntriesLst) == 0 or (len(self.glyphDetailEntriesLst) != self.realNumOfCharactersInImageSegment and len(self.glyphDetailEntriesLst) != self.header().maxEntriesInTableOfDetails) : + print "Error. Fon file load process did not complete correctly. Missing important data in structures. Cannot output image!" + return + + # TODO asdf refine this code here. the dimensions calculation is very crude for now + if self.header().maxGlyphWidth > 0 : + targWidth = (self.header().maxGlyphWidth + paddingBetweenGlyphsX) * (self.realNumOfCharactersInImageSegment + 1) + else: + targWidth = 1080 + + # TODO asdf refine this code here. the dimensions calculation is very crude for now + if self.header().maxGlyphHeight > 0 : + targHeight = self.header().maxGlyphHeight * 2 + else: + targHeight = 480 + + imTargetGameFont = Image.new("RGBA",(targWidth, targHeight), (0,0,0,0)) + #print imTargetGameFont.getbands() + # + # Now fill in the image segment + # Fonts in image segment are stored in pixel colors from TOP to Bottom, Left to Right per GLYPH. + # Each pixel is 16 bit (2 bytes). Highest bit seems to determine transparency (on/off flag). + # There seem to be 5 bits per RGB channel and the value is the corresponding 8bit value (from the 24 bit pixel color) shifting out (right) the 3 LSBs + # First font image is the special character (border of top row and left column) - color of font pixels should be "0x7FFF" for filled and "0x8000" for transparent + drawIdx = 0 + drawIdxDeductAmount = 0 + for idx in range(0, self.realNumOfCharactersInImageSegment): + # TODO check for size > 0 for self.glyphPixelData + # TODO mark glyph OUTLINES? (optional by switch) + (glyphXoffs, glyphYoffs, glyphWidth, glyphHeight, glyphDataOffs) = self.glyphDetailEntriesLst[idx] + glyphDataOffs = glyphDataOffs * 2 + #print idx, glyphDataOffs + currX = 0 + currY = 0 + if (glyphWidth == 0 or glyphHeight == 0): + drawIdxDeductAmount += 1 + drawIdx = idx - drawIdxDeductAmount + + for colorIdx in range(0, glyphWidth*glyphHeight): + tmpTuple = struct.unpack_from('H', self.glyphPixelData, glyphDataOffs) # unsigned short 2 bytes + pixelColor = tmpTuple[0] + glyphDataOffs += 2 + +# if pixelColor > 0x8000: +# print "WEIRD CASE" # NEVER HAPPENS - TRANSPARENCY IS ON/OFF. There's no grades of transparency + rgbacolour = (0,0,0,0) + if pixelColor == 0x8000: + rgbacolour = (0,0,0,0) # alpha: 0.0 fully transparent + else: + tmp8bitR1 = ( (pixelColor >> 10) ) << 3 + tmp8bitG1 = ( (pixelColor & 0x3ff) >> 5 ) << 3 + tmp8bitB1 = ( (pixelColor & 0x1f) ) << 3 + rgbacolour = (tmp8bitR1,tmp8bitG1,tmp8bitB1, 255) # alpha: 1.0 fully opaque + #rgbacolour = (255,255,255, 255) # alpha: 1.0 fully opaque + + if currX == glyphWidth: + currX = 0 + currY += 1 + + imTargetGameFont.putpixel(( (drawIdx + 1) * (self.header().maxGlyphWidth + paddingBetweenGlyphsX ) + currX, paddingFromTopY + glyphYoffs + currY), rgbacolour) + currX += 1 + + imTargetGameFont.save(os.path.join('.', self.simpleFontFileName + ".PNG"), "PNG") + + def header(self): + return self.m_header +# +# +# +if __name__ == '__main__': + # main() + print "Running %s as main module" % (my_module_name) + # assumes a file of name TAHOMA24.FON in same directory + inFONFile = None + #inFONFileName = 'TAHOMA24.FON' # USED IN CREDIT END-TITLES and SCORERS BOARD AT POLICE STATION + #inFONFileName = 'TAHOMA18.FON' # USED IN CREDIT END-TITLES + #inFONFileName = '10PT.FON' # BLADE RUNNER UNUSED FONT? + #inFONFileName = 'KIA6PT.FON' # BLADE RUNNER MAIN FONT + inFONFileName = 'SUBTLS_E.FON' # Subtitles font custom + + errorFound = False + try: + print "Opening %s" % (inFONFileName) + inFONFile = open(os.path.join('.',inFONFileName), 'rb') + except: + errorFound = True + print "Unexpected error:", sys.exc_info()[0] + raise + if not errorFound: + allOfFonFileInBuffer = inFONFile.read() + fonFileInstance = fonFile() + if (fonFileInstance.loadFonFile(allOfFonFileInBuffer, len(allOfFonFileInBuffer), inFONFileName)): + print "FON file loaded successfully!" + fonFileInstance.outputFonToPNG() + else: + print "Error while loading FON file!" + inFONFile.close() +else: + #debug + #print "Running %s imported from another module" % (my_module_name) + pass
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/fontCreator/grabberFromPNG17BR.py b/devtools/blade_runner/subtitles/fontCreator/grabberFromPNG17BR.py new file mode 100644 index 0000000000..a02cd5b7c9 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/grabberFromPNG17BR.py @@ -0,0 +1,1138 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +#------------------------------------------------------------------------------- +# Name: grabberFromPNG15BR +# Purpose: Parse the character fonts from a PNG file in order to create a +# FON file for the Westwood Blade Runner PC game. +# +# Author: antoniou +# +# Created: 16-05-2018 +# Copyright: (c) antoniou 2018 +# Licence: +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +#------------------------------------------------------------------------------- + +# +# Let's assume an Input image with only a row of all character glyphs (no double rows...) +# BLADE RUNNER: +# TODO: A way to differentiate between corrupted TAHOMA18 and extended TAHOMA18 (user created) +# TODO: Maybe instead of removing transparency from glyph ROW image (transparency threshold= 0), use a grey gradient for semitransparent pixels for the cases of TAHOMA fonts (and maybe the subtitle ones too) +# Create the font row in GIMP - add layers for shadows if necessary +# Merge all layers (maintaining an alpha channel for the background) +# Select all and float, which should create a floating selection with all the letters +# Promote that to layer and duplicate it. +# Choose one of the duplicated layers and COLORIFY it to PITCH BLACK +# Set the transparency threshold of THAT black layer to 0 +# Put this layer under the other and merge the visible layers. +# This should get rid of semi-transparent pixels while maintaining the "aliasing" effect. There could be a better way but this should work ok. + + +# TODO: A way to export game fonts to png image (in order to be able to create new expanded fonts keeping glyph consistency!) +# Also override the corruption in TAHOMA18 while exporting +# TODO: Re-Check the order of fonts in (in-game resource font files) TAHOMA18 (stored corrupted) and TAHOMA24 (in good condition). +# TODO: print a warning for mismatch of number of letters in encoding override (or internal) and detected fonts in ROW IMAGE (especially if we expect a double exclamation mark at the start - and we ignoring one of the two) +# TODO: Don't crash if mismatch in detected letter (fewer) in row image vs the letters in ascii list of overrideEncoding file +# TODO: A more detailed readme for this tool and how to use it +# +# DONE: enforce overrideEncoding.txt -- this tool should no longer work without one +# DONE: Test greek subs too +# DONE: Letter fonts should be spaced by TAB when copied into GIMP or other app to create the image row of all character glyphs +# DONE: First character should be repeated in the ROW file (but taken into consideration once) in order to get the pixels for the TAB space between letters (left-start column to left-start column) +# DONE: Use the tab space pixels to calculate the KERNING for each letter (x offset) +# DONE: update the image segment size bytes in the header after having completed populating the image segment +# DONE: allow settin explicit kerning and width addon for cases like i and l characters +# DONE: entrée (boiled dog question) - has an e like goose liver pate --> TESTED +# DONE: Tested ok "si senor" from peruvian lady / insect dealer too! +# DONE: ability to manually set kerning (x-offset) for fonts by letter like a list in parameters or in overrideEncoding.txt } i:1,j:-1,l:1 (no space or white line characters) - POSITIVE OR NEGATIVE VALUES BOTH ADMITTED +# DONE: a value of '-' for this means ignore +# DONE: ability to manually set extra width (additional columns at the end of glyph, with transparent color) for fonts by letter like a list in parameters or in overrideEncoding.txt } i:1,j:2,l:1 - POSITIVE VALUES ONLY +# DONE: make space pixels (var spaceWidthInPixels) into an external param? +# DONE: INFO NOTE IT IS NOT POSSIBLE TO have partial transparency + +import os, sys, shutil +from os import walk, errno +import Image +from struct import * +import re +import os.path +from fonFileLib import * + +company_email = "classic.adventures.in.greek@gmail.com" +app_version = "0.50" +app_name = "grabberFromPNGHHBR" +app_name_spaced = "Extract or create Font Files (.FON) for Blade Runner" + + +supportedMIXInputFiles = ['STARTUP.MIX'] +## 4 font files +supportedExportedFONFiles = ['10PT.FON', 'TAHOMA18.FON', 'TAHOMA24.FON', 'KIA6PT.FON'] + +def calculateFoldHash(strFileName): + i = 0 + hash = 0 + strParam = strFileName.upper() + lenFileName = len(strParam); + while i < lenFileName and i < 12: + groupSum = 0 + # work in groups of 4 bytes + for j in range(0, 4): + # LSB first, so the four letters in the string are re-arranged (first letter goes to lower place) + groupSum >>= 8; + if (i < lenFileName): + groupSum |= (ord(strParam[i]) << 24) + i += 1 + else: # if i >= lenFileName but still haven't completed the four byte loop add 0s + groupSum |= 0 + hash = ((hash << 1) | ((hash >> 31) & 1)) + groupSum + hash &= 0xFFFFFFFF # mask here! + #print (strParam +': ' +''.join('{:08X}'.format(hash))) + return hash + +class grabberFromPNG: + origEncoding = 'windows-1252' + defaultTargetLang = "greek" + defaultTargetEncoding = 'windows-1253' #greek + defaultTargetEncodingUnicode = unicode(defaultTargetEncoding, 'utf-8') + + targetEncoding = 'windows-1253' + targetEncodingUnicode = unicode(targetEncoding, 'utf-8') + + overrideEncodingPath = "" + originalFontName = '' + + BR_GameID = 3 + BR_Desc = 'Blade Runner' + BR_CodeName = 'BLADERUNNER' + BR_DefaultFontFileName = 'SUBTLS_E.FON' + + defaultSpaceWidthInPixelsConst = 0x0007 #0x0008 #0x0006 + spaceWidthInPixels = defaultSpaceWidthInPixelsConst + specialGlyphMode = True + autoTabCalculation = True + + reconstructEntireFont = False # TODO: TRUE!!! + minSpaceBetweenLettersInRowLeftToLeft =0 + minSpaceBetweenLettersInColumnTopToTop = 0 + kerningForFirstDummyFontLetter = 0 + yOffsetForAllGlyphsExceptFirstSpecialGamma = 0 +# deductKerningPixels = 0 + + inputFonMixPath = "" + targetFONFilename = BR_DefaultFontFileName +# origFontFilename="" + origFontPropertiesTxt = "" +# imageOriginalPNG="" + imageRowFilePNG="" + copyFontFileName="" + copyFontPropertiesTxt = "" + copyPNGFileName="" + + lettersFound = 0 + listOfBaselines = [] + listOfWidths = [] + listOfHeights = [] # new for Blade Runner support + listOfLetterBoxes = [] + properListOfLetterBoxes = [] + startColOfPrevFontLetter = 0 # new for Blade Runner support + tabSpaceWidth = 0 + startOfAllLettersIncludingTheExtraDoubleAndWithKern = 0 + maxAsciiValueInEncoding = 0 + + listOfXOffsets = [] # new for Blade Runner support + listOfYOffsets = [] # new for Blade Runner support + + listOfExplicitKerning = [] + listOfWidthIncrements = [] + listOfOutOfOrderGlyphs = [] + + targetLangOrderAndListOfForeignLettersAsciiValues= None + + def initOverrideEncoding(self): + ## + ## FOR INIT PURPOSES!!!! + ## + overrideFailed = True + targetLangOrderAndListOfForeignLettersStrUnicode = None + targetLangOrderAndListOfForeignLettersStr = None + # Read from an override file if it exists. Filename should be overrideEncoding.txt (overrideEncodingTextFile) + if(not self.overrideEncodingPath): + overrideEncodingTextFile = u'overrideEncoding.txt' + relPath = u'.' + self.overrideEncodingPath = os.path.join(relPath,overrideEncodingTextFile) + + if os.access(self.overrideEncodingPath, os.F_OK): + ## debug + #print "Override encoding file found: {0}.".format(self.overrideEncodingPath) + overEncodFile = open(self.overrideEncodingPath, 'r') + linesLst = overEncodFile.readlines() + overEncodFile.close() + if linesLst is None or len(linesLst) == 0: + overrideFailed = True + else: + print "Override Encoding Info: " + involvedTokensLst =[] + del involvedTokensLst[:] # unneeded + for readEncodLine in linesLst: + tmplineTokens = re.findall("[^\t\n]+",readEncodLine ) + for x in tmplineTokens: + involvedTokensLst.append(x) + # print involvedTokensLst + #break #only read first line + for tokenNameKeyPair in involvedTokensLst: + nameKeyTupl = tokenNameKeyPair.split('=', 1) + try: + if len(nameKeyTupl) == 2 and nameKeyTupl[0] == 'targetEncoding' and nameKeyTupl[1] is not None and nameKeyTupl[1] != '-' and nameKeyTupl[1] != '': + self.targetEncodingUnicode = unicode(nameKeyTupl[1], 'utf-8') + self.targetEncoding = unicode.encode("%s" % self.targetEncodingUnicode, self.origEncoding) + elif len(nameKeyTupl) == 2 and nameKeyTupl[0] == 'asciiCharList' and nameKeyTupl[1] is not None and nameKeyTupl[1] != '': + targetLangOrderAndListOfForeignLettersStrUnicode = unicode(nameKeyTupl[1], 'utf-8') + elif len(nameKeyTupl) == 2 and nameKeyTupl[0] == 'explicitKerningList' and nameKeyTupl[1] is not None and nameKeyTupl[1] != '-' and nameKeyTupl[1] != '': + # split at comma, then split at ':' and store tuples of character and explicit kerning + explicitKerningTokenUnicode = unicode(nameKeyTupl[1], 'utf-8') + explicitKerningTokenStr = unicode.encode("%s" % explicitKerningTokenUnicode, self.targetEncoding) + tokensOfExplicitKerningTokenStrList = explicitKerningTokenStr.split(',') + for tokenX in tokensOfExplicitKerningTokenStrList: + tokensOfTupleList = tokenX.split(':') + self.listOfExplicitKerning.append((ord(tokensOfTupleList[0]), int(tokensOfTupleList[1])) ) + elif len(nameKeyTupl) == 2 and nameKeyTupl[0] == 'explicitWidthIncrement' and nameKeyTupl[1] is not None and nameKeyTupl[1] != '-' and nameKeyTupl[1] != '': + # split at comma, then split at ':' and store tuples of character and explicit additional width - POSITIVE VALUES ONLY + explicitWidthIncrementTokenUnicode = unicode(nameKeyTupl[1], 'utf-8') + explicitWidthIncrementTokenStr = unicode.encode("%s" % explicitWidthIncrementTokenUnicode, self.targetEncoding) + tokensOfWidthIncrementStrList = explicitWidthIncrementTokenStr.split(',') + for tokenX in tokensOfWidthIncrementStrList: + tokensOfTupleList = tokenX.split(':') + self.listOfWidthIncrements.append((ord(tokensOfTupleList[0]), int(tokensOfTupleList[1]))) + elif len(nameKeyTupl) == 2 and nameKeyTupl[0] == 'specialOutOfOrderGlyphsUTF8ToAsciiTargetEncoding' and nameKeyTupl[1] is not None and nameKeyTupl[1] != '-' and nameKeyTupl[1] != '': + # split at comma, then split at ':' and store tuples of character + explicitOutOfOrderGlyphsTokenUnicode = unicode(nameKeyTupl[1], 'utf-8') # unicode(nameKeyTupl[1], 'utf-8') + #explicitOutOfOrderGlyphsTokenStr = unicode.encode("%s" % explicitOutOfOrderGlyphsTokenUnicode, self.targetEncoding) + #explicitOutOfOrderGlyphsTokenStr = explicitOutOfOrderGlyphsTokenUnicode.decode(self.targetEncoding) # unicode.encode("%s" % explicitOutOfOrderGlyphsTokenUnicode, 'utf-8') + tokensOfOutOfOrderGlyphsStrList = explicitOutOfOrderGlyphsTokenUnicode.split(',') + for tokenX in tokensOfOutOfOrderGlyphsStrList: + tokensOfTupleList = tokenX.split(':') + self.listOfOutOfOrderGlyphs.append( (unichr(ord(tokensOfTupleList[0])), unichr(ord(tokensOfTupleList[1]))) ) + elif len(nameKeyTupl) == 2 and nameKeyTupl[0] == 'originalFontName' and nameKeyTupl[1] is not None and nameKeyTupl[1] != '-' and nameKeyTupl[1] != '': + self.originalFontName = nameKeyTupl[1] + except: + overrideFailed = True + raise + if not (self.targetEncoding is None or self.targetEncoding == '' or targetLangOrderAndListOfForeignLettersStrUnicode is None or len(targetLangOrderAndListOfForeignLettersStrUnicode) == 0 or self.originalFontName is None or self.originalFontName == ''): + overrideFailed = False + + print "Target Encoding: " , self.targetEncoding + #print "Lang Order: " , targetLangOrderAndListOfForeignLettersStrUnicode + print "Explicit Kern List: " , self.listOfExplicitKerning + print "Explicit Width Increment List: " , self.listOfWidthIncrements + print "Original Font Name: " , self.originalFontName + + if(len(self.listOfOutOfOrderGlyphs) == 0 and self.specialGlyphMode == True): + # Just keep those that are needed + if self.originalFontName == 'SUBTLS_E': + self.listOfOutOfOrderGlyphs.append((u'\xed', u'\u0386')) # spanish i (si) + self.listOfOutOfOrderGlyphs.append((u'\xf1', u'\xa5')) # spanish n (senor) + self.listOfOutOfOrderGlyphs.append((u'\xe2', u'\xa6')) # a for (liver) pate + self.listOfOutOfOrderGlyphs.append((u'\xe9', u'\xa7')) # e for (liver) pate + elif self.originalFontName == 'TAHOMA': # treat TAHOMA18 and TAHOMA24 similarily here + self.listOfOutOfOrderGlyphs.append((u'\xe9', u'\u0192')) # french e punctuated + self.listOfOutOfOrderGlyphs.append((u'\xfc', u'\u2013')) # u umlaut + print "Explicit Out Of Order Glyphs List: " , self.listOfOutOfOrderGlyphs + else: + ## debug + print "Error: Override encoding file not found: {0}.".format(self.overrideEncodingPath) + #print "Override encoding file not found: {0}.".format(self.overrideEncodingFileRelPath) + #print "To override the default encoding {0} use an override encoding file with two tab separated entries: encoding (ascii) and characters-list. Convert to UTF-8 without BOM and save. For example:".format(defaultTargetEncoding) + #print "windows-1252\t!!\"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}" + pass + + if overrideFailed: + ## debug + print "Override encoding file FAILED-1." #" Initializing for {0}...".format(defaultTargetLang) + # self.targetEncoding = self.defaultTargetEncoding + # self.targetEncodingUnicode = self.defaultTargetEncodingUnicode + # targetLangOrderAndListOfForeignLettersStrUnicode = unicode(allOfGreekChars, 'utf-8') + # #print targetLangOrderAndListOfForeignLettersStrUnicode + sys.exit() # terminate if override Failed (Blade Runner) + + try: + targetLangOrderAndListOfForeignLettersStr = unicode.encode("%s" % targetLangOrderAndListOfForeignLettersStrUnicode, self.targetEncoding) + except: + ## debug + print "Override encoding file FAILED-2." #"Initializing for {0}...".format(defaultTargetLang) + # self.targetEncoding = self.defaultTargetEncoding + # self.targetEncodingUnicode = self.defaultTargetEncodingUnicode + # targetLangOrderAndListOfForeignLettersStrUnicode = unicode(allOfGreekChars, 'utf-8') + # targetLangOrderAndListOfForeignLettersStr = unicode.encode("%s" % targetLangOrderAndListOfForeignLettersStrUnicode, self.targetEncoding) + # #print targetLangOrderAndListOfForeignLettersStrUnicode + sys.exit() # terminate if override Failed (Blade Runner) + + targetLangOrderAndListOfForeignLetters = list(targetLangOrderAndListOfForeignLettersStr) + print targetLangOrderAndListOfForeignLetters, len(targetLangOrderAndListOfForeignLetters) # new + self.targetLangOrderAndListOfForeignLettersAsciiValues = [ord(i) for i in targetLangOrderAndListOfForeignLetters] + print self.targetLangOrderAndListOfForeignLettersAsciiValues, len(self.targetLangOrderAndListOfForeignLettersAsciiValues) + self.maxAsciiValueInEncoding = max(self.targetLangOrderAndListOfForeignLettersAsciiValues) + +# for charAsciiValue in targetLangOrderAndListOfForeignLetters: +# print "Ord of chars: %d" % ord(charAsciiValue) + + ## + ## END OF INIT CODE + ## + +# +# TODO: warning: assumes that there is a margin on top and bellow the letters +# (especially on top; if a letter starts from the first pixel row, +# it might not detect it!) <---to fix +# + def __init__(self, pselectedEncoding=None): + self.minSpaceBetweenLettersInRowLeftToLeft = 0 + self.minSpaceBetweenLettersInColumnTopToTop = 0 + self.kerningForFirstDummyFontLetter = 0 + self.yOffsetForAllGlyphsExceptFirstSpecialGamma = 0 + self.spaceWidthInPixels = self.defaultSpaceWidthInPixelsConst +# self.deductKerningPixels = 0 + self.reconstructEntireFont = False # TODO : True? + #self.origFontFilename=porigFontFilename + self.inputFonMixPath = "" + self.targetFONFilename = self.BR_DefaultFontFileName + self.copyFontFileName = "" + self.copyPNGFileName="" + #self.imageOriginalPNG=pimageOriginalPNG + self.imageRowFilePNG = "" + self.baselineOffset = 0 + + self.lettersFound = 0 + self.origFontPropertiesTxt = "" + self.copyFontPropertiesTxt = "" + self.cleanup() # for good practice (this should not be here, but it's here too due to quick/sloppy :) coding (TODO: fix it) + #debug + #self.DBinit() # TODO REMOVE? + if pselectedEncoding == None: + pselectedEncoding = self.targetEncoding + + self.selectedGameID = self.BR_GameID + self.activeEncoding = pselectedEncoding + self.lettersInOriginalFontFile = 0 #initialization + + # TODO: we should get from the DB the encoding and lettersString + # and the Empty slots for the selected Game and Calculate the rest of + # the lists/dictionaries on the fly. + # IF no lettersString or encoding has been defined... + # then issue error? or continue with hard-coded + # (insert to db as well and inform the GUI?) + #self.calcFromDB() # TODO REMOVE? + return + + + def cleanup(self): + self.lettersFound = 0 + self.startColOfPrevFontLetter = 0 + self.tabSpaceWidth = 0 + self.startOfAllLettersIncludingTheExtraDoubleAndWithKern = 0 + del self.listOfBaselines[:] + del self.listOfWidths[:] + del self.listOfHeights[:] + del self.listOfLetterBoxes[:] + del self.properListOfLetterBoxes[:] + del self.listOfXOffsets[:] # new for Blade Runner support + del self.listOfYOffsets[:] # new for Blade Runner support +# del self.listOfExplicitKerning[:] # don't clean these up +# del self.listOfWidthIncrements[:] # don't clean these up + + self.origFontPropertiesTxt = "" + self.copyFontPropertiesTxt = "" + return + +## +## SETTERS +## + def setInputPathForFonMix(self, pInputMixPath): + self.inputFonMixPath = pInputMixPath + return + + def setImageRowFilePNG(self, pimageRowFilePNG): + self.imageRowFilePNG = pimageRowFilePNG + return + + def setTargetFONFilename(self, pTargetFONFilename): + self.targetFONFilename = pTargetFONFilename + return + + def setMinSpaceBetweenLettersInRowLeftToLeft(self, pminSpaceBetweenLettersInRowLeftToLeft): + self.minSpaceBetweenLettersInRowLeftToLeft = pminSpaceBetweenLettersInRowLeftToLeft + return + def setMinSpaceBetweenLettersInColumnTopToTop(self, pminSpaceBetweenLettersInColumnTopToTop): + self.minSpaceBetweenLettersInColumnTopToTop = pminSpaceBetweenLettersInColumnTopToTop + return + + def setKerningForFirstDummyFontLetter(self, pKerningForFirstDummyFontLetter): + self.kerningForFirstDummyFontLetter = pKerningForFirstDummyFontLetter + return + + def setYOffsetForAllGlyphsExceptFirstSpecialGamma(self, pYOffsetForAllGlyphsExceptFirstSpecialGamma): + self.yOffsetForAllGlyphsExceptFirstSpecialGamma = pYOffsetForAllGlyphsExceptFirstSpecialGamma + return + +# def setDeductKerningPixels(self, pDeductKerningPixels): +# self.deductKerningPixels = pDeductKerningPixels +# return + + def setSpaceWidthInPixels(self, pSpaceWidthInPixels): + self.spaceWidthInPixels = pSpaceWidthInPixels + return + + def setSpecialGlyphMode(self, pSpecialGlyphMode): + self.specialGlyphMode = pSpecialGlyphMode + return + + + def setAutoTabCalculation(self, pAutoTabCalculation): + self.autoTabCalculation = pAutoTabCalculation + return + + def setOverrideEncodingPath(self, pOverrideEncodingPath): + self.overrideEncodingPath = pOverrideEncodingPath + return + +## +## END OF SETTERS +## + + + + def parseImage(self, loadedImag, imwidth, imheight, trimTopPixels=0, trimBottomPixels = 0, firstDoubleLetterIgnore = False): + """ parsing input image and detect one character font per run, and deleting the detected character font after calculating its specs (this is done in-memory; we are not writing back to the file) + """ + prevColStartForLetter = 0 + prevRowStartForLetter = 0 + startCol = 0 + startRow = 0 + endCol = 0 + endRow = 0 + for x in range(0, imwidth): # for each column + if startCol != 0: + break + for y in range(0, imheight): # we search all rows (for each column) + r1,g1,b1,a1 = loadedImag[x, y] + if a1 != 0: # if pixel not completely transparent -- this is not necessarily the *top* left pixel of a font letter though! -- the startRow is still to be determined. + # print loadedImag[x, y] + if prevColStartForLetter == 0: + prevColStartForLetter = x + prevRowStartForLetter = y + startCol = x + # print "Letter found" + # print "start col:%d" % startCol + # # print "hypothe row:%d" % y + # # starting from the first row of the row-image (to do optimize), we parse by rows to find the top point (row coordinate) of the character font + # for y2 in range(0, y+1): + tmpSum = y + self.minSpaceBetweenLettersInColumnTopToTop + scanToRow = imheight # - explicitly set this to the whole image height -- assumes only one row of character fonts + if tmpSum < imheight: # TODO: WAS scanToRow < imheight but this doesn't get executed anymore due to explicitly setting scanToRow to imheight (assuming only one row of character fonts) + # DONE: NEW changed check to if tmpSum < imheight which makes more sense + scanToRow = tmpSum + for y2 in range(0, scanToRow): # a loop to find the startRow - Check by row (starting from the top of the image and the left column of the letter) + if startRow != 0: + break + tmpSum = startCol + self.minSpaceBetweenLettersInRowLeftToLeft + scanToCol = imwidth + if tmpSum < imwidth: + scanToCol = tmpSum + #print (startCol, scanToCol) + for x2 in range(startCol, scanToCol): # check all columns (for each row) + #print loadedImag[x2, y2] + r2,g2,b2,a2 = loadedImag[x2, y2] + if a2 != 0 and startRow == 0: + startRow = y2 + trimTopPixels + # print "start row: %d" % startRow + break + if startCol > 0 and startRow > 0: # WARNING: TODO NOTE: SO NEVER HAVE AN INPUT IMAGE WHERE THE FONT CHARACTERS ARE TOUCHING THE TOP OF THE IMAGE WITH NO EMPTY SPACE WHATSOEVER + tmpSum = startRow + self.minSpaceBetweenLettersInColumnTopToTop + scanToRow = imheight + if tmpSum < imheight: + scanToRow = tmpSum + tmpSum = startCol + self.minSpaceBetweenLettersInRowLeftToLeft + scanToCol = imwidth + if tmpSum < imwidth: + scanToCol = tmpSum + for y in range(startRow, scanToRow): # now check per row (we go through all theoretical rows, no breaks)-- we want to find the bottom row + for x in range(startCol, scanToCol): # check the columns for each row + r1,g1,b1,a1 = loadedImag[x, y] + if a1 != 0: + endRow = y + if endRow > 0: + endRow = endRow - trimBottomPixels + # print "end row:% d" %endRow + + if startCol > 0 and startRow > 0 and endRow > 0: + tmpSum = startCol + self.minSpaceBetweenLettersInRowLeftToLeft + scanToCol = imwidth + if tmpSum < imwidth: + scanToCol = tmpSum + for x in range(startCol, scanToCol): # now check per column (we go through all theoretical columns, no breaks) -- we want to find the bottom column + for y in range(startRow, endRow+1): # check the rows for each column + r1,g1,b1,a1 = loadedImag[x, y] + #print loadedImag[x, y] + if a1 != 0: + endCol = x + # print "end col:% d" %endCol + if startCol > 0 and startRow > 0 and endRow > 0 and endCol > 0: + # append deducted baseline + # + if firstDoubleLetterIgnore == True: + self.startOfAllLettersIncludingTheExtraDoubleAndWithKern = startCol - self.kerningForFirstDummyFontLetter + else: # firstDoubleLetterIgnore == False + if self.autoTabCalculation == True: + if self.tabSpaceWidth == 0: + #print "start startPre", startCol, self.startColOfPrevFontLetter + self.tabSpaceWidth = startCol - self.startColOfPrevFontLetter + print "Tab Space Width detected: %d " % (self.tabSpaceWidth) + # new if -- don't use else here, to include the case of when we first detected the tab space width + if self.tabSpaceWidth > 0: + self.listOfXOffsets.append(startCol - (self.startOfAllLettersIncludingTheExtraDoubleAndWithKern + (self.lettersFound + 1) * self.tabSpaceWidth) ) # + self.deductKerningPixels ) + #print "xOffSet", startCol - (self.startOfAllLettersIncludingTheExtraDoubleAndWithKern + (self.lettersFound + 1) * self.tabSpaceWidth) + else: + self.listOfXOffsets.append(0) + + + + self.listOfBaselines.append(endRow) + self.listOfWidths.append(endCol-startCol + 1) # includes the last col (TODO this was without the +1 for MI:SE translator -- possible bug? did we compensate?) + self.listOfHeights.append(endRow - startRow + 1) # +1 includes the last row + self.listOfLetterBoxes.append((startCol, startRow, endCol, endRow)) + + self.startColOfPrevFontLetter = startCol #update for next pass + #delete the letter - even in the case of ignoring the first double letter + for x in range(startCol, endCol+1): + for y in range(startRow - trimTopPixels, endRow+1 + trimBottomPixels): + loadedImag[x, y] = 0, 0, 0, 0 + return 0 + else: return -1 +# +# +# + def generateModFiles(self, customBaselineOffs): + """ Generate font (FON) files (work on copies, not the originals). Return values: 0 no errors, -1 output font file has alrady new letters, -2 no fonts found in png (TODO: more error cases) + """ + # + # When a customBaselineOffs is set, we should expand the space for the letter (otherwise it will be overflown in the next line or truncated (outside the png) + # We can't expand the space for the letter downwards, because the engine will (com)press the new height to fit its expected and it will look bad. + # NO THAT WON'T WORK--> The font will remain in the wrong place: Should we CUT from the top and hope that we don't trunctate!? Keeping the resulting height equal to the expected one? + # MAYBE: Do not alter the baseline of the original file, but the detected (popular) one of the line file!!! + retVal = 0 + totalFontLetters = 0 + importedNumOfLetters = 0 + errMsg = "" + errorFound = False + im = None + pix = None + pixReloaded = None + # + # CONSTANTS + # + origGameFontDiakenoHeight = 0 + interLetterSpacingInPNG = 4 + origGameFontSizeEqBaseLine = 0 + # offset for start of PNG index table +# firstTableLineOffset = self.PNG_TABLE_STARTLINE_OFFSET + lettersInOriginalFontFile = self.lettersInOriginalFontFile + # + # detection of origGameFontSizeEqBaseLine + # + #origGameFontSizeEqBaseLine = self.findDetectedBaseline() ## NEW BR REMOVED + self.cleanup() # necessary after detection of baseline, because it fills up some of the lists used in the following! + +## self.origFontPropertiesTxt = self.getImagePropertiesInfo(True) # "%dx%dx%s" % (im.size[0],im.size[1], im.mode) # NEW REMOVED +# print "WEEEE::: ", self.imageOriginalPNG, im.format, "%dx%d" % im.size, im.mode +# print "BASELINE DETECTED:%d " % origGameFontSizeEqBaseLine + + # + # OPEN THE IMAGE WITH THE ROW OF CHARACTER FONTS TO BE IMPORTED + # + if os.access(self.imageRowFilePNG, os.F_OK) : + try: + im = Image.open(self.imageRowFilePNG) + except: + errMsg = "No letters were found in input png - IO exception!" + print errMsg + retVal = -2 + errorFound = True + else: + errMsg = "No letters were found in input png - IO fault!" + print errMsg + retVal = -2 + errorFound = True + if not errorFound: + #debug + #print self.imageRowFilePNG, im.format, "%dx%d" % im.size, im.mode + w1, h1 = im.size + trimTopPixels = 0 + trimBottomPixels = 0 + italicsMode = False # will be set to true only if the prefix of the row file is itcrp_ or it_ in order to activate some extra settings for kerning and letter width! + # TODO the note about special handling of row PNG files with it_ or itcrp_ prefix, should be moved to the documentation + # TODO the special settings for handling italic native letters should be in the settings(?) + filepathSplitTbl = os.path.split(self.imageRowFilePNG) + sFilenameOnlyImageRowFilePNG = filepathSplitTbl[1] + + if sFilenameOnlyImageRowFilePNG.startswith("itcrp_") or sFilenameOnlyImageRowFilePNG.startswith("it_"): + italicsMode = True + + if sFilenameOnlyImageRowFilePNG.startswith("itcrp_"): + trimTopPixels = 1 + trimBottomPixels = 1 + print "Will trim upper line by %d pixels and bottom line by %d pixels" % (trimTopPixels, trimBottomPixels) + pix = im.load() + # pix argument is mutable (will be changed in the parseImage body) + if self.parseImage(pix, w1, h1, trimTopPixels, trimBottomPixels, True) == 0: #first run, just get the start column, ignore the letter - don't store it . We need this for the tab space width calculation and eventually the kerning calc of the letters + # after the first call, we got an update on self.startColOfPrevFontLetter using the dummy double firstg letter font + while self.parseImage(pix, w1, h1, trimTopPixels, trimBottomPixels) == 0: + self.lettersFound = self.lettersFound + 1 # == 0 means one character font was detected so +1 to the counter + # print self.listOfBaselines + #debug + print "Font Letters Detected (not including the first double): %d" % (self.lettersFound) + if (self.lettersFound ) > 0 : + #print "widths: ", self.listOfWidths[0:] + #print "Plain x offsets:" + #print zip(self.targetLangOrderAndListOfForeignLettersAsciiValues[1:], self.listOfXOffsets) +# # normalize x offsets +# minXoffset = min(self.listOfXOffsets) +# if(minXoffset < 0): +# addNormalizer = minXoffset * (-1) +# self.listOfXOffsets = [ x + addNormalizer for x in self.listOfXOffsets] +# print "Normalized x offsets: " +# print self.listOfXOffsets + # calculate y offsets + (listOfStartCols, listOfStartRows, listOfEndCols, listOfEndRows) = zip(* self.listOfLetterBoxes) + minTopRow = min(listOfStartRows) + self.listOfYOffsets = [ x - minTopRow for x in listOfStartRows] + if (self.yOffsetForAllGlyphsExceptFirstSpecialGamma != 0): + self.listOfYOffsets = [ x + self.yOffsetForAllGlyphsExceptFirstSpecialGamma for x in self.listOfYOffsets] + #print "Y offsets: " + #print self.listOfYOffsets + # + # + # # actually explicit Width setting could affect this so calc a new list here with final widths and get the max on that list! + # + listOfCalcWidths = [] + kIncIndx = 1 + for tmpWidth in self.listOfWidths: + explicitWidthIncrementVal = 0 + if len(self.listOfWidthIncrements ) > 0: + tmpOrd = self.targetLangOrderAndListOfForeignLettersAsciiValues[kIncIndx] + keysOfWidthIncrements, valuesOfWidthIncrements = (zip(*self.listOfWidthIncrements)) + if tmpOrd in keysOfWidthIncrements: + print "Explicit width increment for %d: %d" % (tmpOrd, valuesOfWidthIncrements[keysOfWidthIncrements.index(tmpOrd)]) + explicitWidthIncrementVal = valuesOfWidthIncrements[keysOfWidthIncrements.index(tmpOrd)] + listOfCalcWidths.append(tmpWidth + explicitWidthIncrementVal ) + if explicitWidthIncrementVal == 0: + listOfCalcWidths.append(tmpWidth) + kIncIndx = kIncIndx + 1 + #maxFontWidth = max(self.listOfWidths) + maxFontWidth = max(listOfCalcWidths) + maxFontHeight = max(self.listOfHeights) + print "Max Width, Max Height (not necessarily for the same character font): %d, %d" % (maxFontWidth, maxFontHeight) + #print "Index\tAsciiOrd\tX Offs\tY Offs\tWidth\tHeight" + #print zip(range(1, len(self.listOfXOffsets)), self.targetLangOrderAndListOfForeignLettersAsciiValues[1:], self.listOfXOffsets, self.listOfYOffsets, listOfCalcWidths, self.listOfHeights) + targetFontFile = None + try: + targetFontFile = open(self.targetFONFilename, 'wb') + except: + errorFound = True + if not errorFound: + # reopen the image with our Fonts because we deleted the letters in the in-mem copy + im = None + if os.access(self.imageRowFilePNG, os.F_OK) : + try: + im = Image.open(self.imageRowFilePNG) + except: + errorFound = True + else: + errorFound = True + if not errorFound: + pixReloaded = None + pixReloaded = im.load() + + # first 4 bytes are the max ascii char value supported (it's basically the number of entries in the character index table) + # next 4 bytes are max font char width (pixels) + # next 4 bytes are max font char height (pixels) + # next 4 bytes give the size of the graphic segment for the font characters (this size is in word units, so it needs *2 to get the byte size) + # this size should be updated at the end (after filling the file with all font image data) + # + # pack 'I' unsigned int + print "NumberOfEntriesInFontTabl", (self.maxAsciiValueInEncoding + 1 + 1) + numberOfEntriesInFontTable = self.maxAsciiValueInEncoding + 1 + 1 # 0x0100 # This is actually the max ascii value + plus one (1) to get the font index value + plus another one (1) to get the count (since we have zero based indices) + # TODO ??? could be more than this if we need to keep other characters (not in our codeset) and expand the ascii table and offset the new characters + numberOfEntriesInFontTableInFile = pack('I', numberOfEntriesInFontTable ) + targetFontFile.write(numberOfEntriesInFontTableInFile) + maxFontWidthPixelsToWrite = pack('I', maxFontWidth) + targetFontFile.write(maxFontWidthPixelsToWrite) + maxFontHeightPixelsToWrite = pack('I', maxFontHeight) + targetFontFile.write(maxFontHeightPixelsToWrite) + fontImagesSegmentSize = pack('I', 0x0000) # - to be updated at the end! + targetFontFile.write(fontImagesSegmentSize) + + startOfImageSegmentAbs = 0x10 + 20 * numberOfEntriesInFontTable # header is 0x10 bytes. Then table of 20 bytes * numberOfEntriesInFontTable and then the data. + lastImageSegmentOffset = 0 + # targetFontFile.close() # don't close here + # + # Fonts index table - properties and offset in image segment + # TODO - REVISE WHEN FINISHED WITH COMPLETE TRANSCRIPT for special glyphs + # So far additional required characters (not included in the standard ASCII (127 chars) are: + # the spanish i (put it in ASCII value 0xA2 (162), font index 0xA3)? todo verify -- actual ASCII value in codepage 1252 is 0xED (237) + # the spanish n (put it in ASCII value 0xA5 (165), font index 0xA6)? todo verify -- actual ASCII value in codepage 1252 is 0xF1 (241) + # DONE we also need special fonts for liver pâté + # a actual ASCII value is 0xE2 (226) in codepage 1252 -- put it in ASCII value 0xA6 (166) -- font index 0xA7 + # e actual ASCII value is 0xE9 (233) in codepage 1252 -- put it in ASCII value 0xA7 (167) -- font index 0xA8 + # In the row png font glyphs, the letter glyphs (images) should be the actual character fonts (spanish n, i etc) + # but in the overrideEncoding.txt we need the corresponding ASCII characters for the particular codepage of the text (eg here the greek windows-1253) + # + # NOTE! WARNING: We need to add the corresponding ASCII characters for our codepage (eg for Windows 1253 the characters with value 0xA2 and 0xA5 which are not the spanish characters but will act as delegates for them) + # the greek Ά (alpha tonoumeno) character has ASCII value 0xA2 (162) (in codeset Windows 1253) so conflict with spanish i in in-game Tahoma -- put it in 0xA3 (163) (font index 0xA4) + # We should fill all unused characters with "space" placeholder. Probably all of them should also point to the same area (second block) of the image segment too. + # First block of the image area (font index = 0) is reserved and should be the "border" gamma-like character. + # + # Kerning of the first letter font is '1' for Tahoma18 (when shadowed from every side (the left side shadow reduces the kerning), otherwise it would be 2) -- TODO for now this should be a launch parameter + # Y offset should be calculated from the top row of the heighest font + #kIncIndx = 0 + ## aux list because the order of the fonts in our overrideEncoding may not match the real order in the ascii table + #listOfWriteableEntries = [ (0,0,0,0,0) for i in range(0, numberOfEntriesInFontTable)] # create a list of placeholders for the number of entries we need to write + #print " *************** DBG **************" + #print listOfWriteableEntries + #print " *************** DBG **************" + del self.properListOfLetterBoxes[:] + for i in range(0, numberOfEntriesInFontTable): # blocks of 20 bytes + # 20 byte block + # 4 bytes x offset (from what ref point? is this for kerning ) - CAN THIS BE NEGATIVE? + # 4 bytes y offset (from what ref point? is this for the baseline?) - CAN THIS BE NEGATIVE? + # 4 bytes char width + # 4 bytes char height + # 4 bytes offset in image segment (units in words (2 bytes)) + + # TODO add all standard ascii characters in the ROW IMAGE before the additional required spanish and then GREEK alphabet characters -- + # -- greek Ά should be at its proper place (between spanish i and spanish n). + # TODO check possible support issues for ώ greek character + if i == 0: + # the first entry is a special font character of max width and max height with a horizontal line across the top-most row and a vertical line across the left-most column + tmpXOffsetToWrite = pack('I', 0x0000) + targetFontFile.write(tmpXOffsetToWrite) + tmpYOffsetToWrite = pack('I', 0x0000) + targetFontFile.write(tmpYOffsetToWrite) + tmpWidthToWrite = pack('I', maxFontWidth) + targetFontFile.write(tmpWidthToWrite) + tmpHeightToWrite = pack('I', maxFontHeight) + targetFontFile.write(tmpHeightToWrite) + tmpDataOffsetToWrite = pack('I', 0x0000) # start of image segment means 0 offset + targetFontFile.write(tmpDataOffsetToWrite) + # TODO maybe conform more with game's format: Eg Tahoma24.fon (native game resource) does not always point to the second character font offset for dummy entries, but to the latest offset and only additionally sets the x-offset property (all others are 0) - eg look for 0x74c9 offsets (byte sequence 0xc9 0x74) + dummyCharFontImageConstOffset = maxFontWidth * maxFontHeight; # const. actual offset in bytes is twice that. This counts in words (2-bytes) - This points to the first valid entry but with properties that make it translate as a space or dummy(?) + lastImageSegmentOffset = maxFontWidth * maxFontHeight; # actual offset in bytes is twice that. This counts in words (2-bytes) + #listOfWriteableEntries[0] = (tmpXOffsetToWrite, tmpYOffsetToWrite, tmpWidthToWrite, tmpHeightToWrite, tmpDataOffsetToWrite) + else: + if (i-1) in self.targetLangOrderAndListOfForeignLettersAsciiValues: + # then this is an actual entry + # i-1 is the order of an ascii character, that should be placed in the next slot in the output FON file + # but this ascii character in the input overrideEncoding could be not it the same i spot -- because of the correspondance to an out-of-order PNG row file + kIncIndxLst = [item for item in enumerate(self.targetLangOrderAndListOfForeignLettersAsciiValues[1:], 0) if item[1] == (i-1) ] + kIncIndx = kIncIndxLst[0][0] + #kIncIndx = self.targetLangOrderAndListOfForeignLettersAsciiValues.index(i-1) + #print kIncIndxLst + #print kIncIndx, i-1 + #print i, ": actual entry index of ascii char", (i-1)," width:", self.listOfWidths[kIncIndx] + #print "Self explicit kerning list: " , self.listOfExplicitKerning + if len(self.listOfExplicitKerning ) > 0: + keysOfExplicitKerning, valuesOfExplicitKerning = (zip(*self.listOfExplicitKerning)) + if (i - 1) in keysOfExplicitKerning: + # found explicit kerning for this + print "Explicit kerning for %d " % (i-1) + self.listOfXOffsets[kIncIndx] = valuesOfExplicitKerning[keysOfExplicitKerning.index(i-1)] # explicit X offset + + tmpXOffsetToWrite = pack('i', self.listOfXOffsets[kIncIndx]) # x offset - from left # TODO check if ok. Changed to signed int since it can be negative sometimes! + targetFontFile.write(tmpXOffsetToWrite) + tmpYOffsetToWrite = pack('I', self.listOfYOffsets[kIncIndx]) # y offset from topmost row + targetFontFile.write(tmpYOffsetToWrite) + + if len(self.listOfWidthIncrements ) > 0: + keysOfWidthIncrements, valuesOfWidthIncrements = (zip(*self.listOfWidthIncrements)) + if (i - 1) in keysOfWidthIncrements: + print "Explicit width increment for %d " % (i-1) + foundExplicitWidthIncrement = True + self.listOfWidths[kIncIndx] = self.listOfWidths[kIncIndx] + valuesOfWidthIncrements[keysOfWidthIncrements.index(i-1)] + + tmpWidthToWrite = pack('I', self.listOfWidths[kIncIndx] ) + targetFontFile.write(tmpWidthToWrite) + tmpHeightToWrite = pack('I', self.listOfHeights[kIncIndx]) + targetFontFile.write(tmpHeightToWrite) + tmpDataOffsetToWrite = pack('I', lastImageSegmentOffset) # + targetFontFile.write(tmpDataOffsetToWrite) + lastImageSegmentOffset = lastImageSegmentOffset + self.listOfWidths[kIncIndx] * self.listOfHeights[kIncIndx] + #kIncIndx = kIncIndx + 1 # increases only for valid characters + # + # populate self.properListOfLetterBoxes here + # + self.properListOfLetterBoxes.append(self.listOfLetterBoxes[kIncIndx]) + else: + # + #print i, ": phony entry" + # TODO in-game resource fonts don't point all to the first entry as dummy but to the last valid entry encountered + tmpXOffsetToWrite = pack('I', 0x0000) # 0 x offset + targetFontFile.write(tmpXOffsetToWrite) + tmpYOffsetToWrite = pack('I', 0x0000) # 0 y offset + targetFontFile.write(tmpYOffsetToWrite) + tmpWidthToWrite = pack('I', self.spaceWidthInPixels) # font width set for some pixels of space + targetFontFile.write(tmpWidthToWrite) + tmpHeightToWrite = pack('I', 0x0000) + targetFontFile.write(tmpHeightToWrite) + tmpDataOffsetToWrite = pack('I', dummyCharFontImageConstOffset) # + targetFontFile.write(tmpDataOffsetToWrite) + # end of for loop over all possible ascii values contained in the fon file + # print the corrected properties per glyph font: + print "***** FINAL (Explicit Kern, width accounted) *****\nIndex\tAsciiOrd\tX Offs\tY Offs\tWidth\tHeight" + tmpListOfTuplesToPrintDbg = zip(range(1, len(self.listOfXOffsets)), self.targetLangOrderAndListOfForeignLettersAsciiValues[1:], self.listOfXOffsets, self.listOfYOffsets, listOfCalcWidths, self.listOfHeights) + for itemDbg in tmpListOfTuplesToPrintDbg: + print "%4d\t%8d\t%6d\t%6d\t%6d\t%6d" % (itemDbg[0], itemDbg[1], itemDbg[2], itemDbg[3], itemDbg[4], itemDbg[5]) + + # + # + # Now fill in the image segment + # Fonts are written from TOP to Bottom, Left to Right. Each pixel is 16 bit (2 bytes). Highest bit seems to determine transparency (on/off flag). + # + # There seem to be 5 bits per RGB channel and the value is the corresponding 8bit value (from the 24 bit pixel color) shifting out (right) the 3 LSBs + # NOTE: Since we can't have transparency at channel level(?), it's best to have the input PNG not have transparent colored pixels (in Gimp merge the font layers, foreground and shadow and then from Layer settings set transparency threshold to 0 for that layer)- keep the background transparent! + # + # First font image is the special character (border of top row and left column) - color of font pixels should be "0x7FFF" for filled and "0x8000" for transparent + # + # + # Then follow up with the image parts for each letter! + # + # + # + # START of First special character image segment + # + for i in range(0, maxFontWidth * maxFontHeight): + if(i < maxFontWidth or i % maxFontWidth == 0): + tmpPixelColorRGB555ToWrite = pack('H', 0x7FFF) #unsigned short - 2 bytes + targetFontFile.write(tmpPixelColorRGB555ToWrite) + else: + tmpPixelColorRGB555ToWrite = pack('H', 0x8000) + targetFontFile.write(tmpPixelColorRGB555ToWrite) # unsigned short - 2 bytes + # + # END of First special character image segment + # + # + # Start rest of the font characters image segments + # + # + + # + # If we have a character with explicit width increment (y) we should add columns of transparent colored pixels at the end (so since this is done by row, we should add y number of transparent pixels at the end of each row) + kIncIndx = 1 # start after the first glyph (which is DOUBLE) + for (c_startCol, c_startRow, c_endCol, c_endRow) in self.properListOfLetterBoxes[0:]: + #print (c_startCol, c_startRow, c_endCol, c_endRow),'for letter ', self.targetLangOrderAndListOfForeignLettersAsciiValues[kIncIndx] + explicitWidthIncrementVal = 0 + if len(self.listOfWidthIncrements ) > 0: + tmpOrd = self.targetLangOrderAndListOfForeignLettersAsciiValues[kIncIndx] + keysOfWidthIncrements, valuesOfWidthIncrements = (zip(*self.listOfWidthIncrements)) + if tmpOrd in keysOfWidthIncrements: + #print "Explicit width increment for %d: %d" % (tmpOrd, valuesOfWidthIncrements[keysOfWidthIncrements.index(tmpOrd)]) + explicitWidthIncrementVal = valuesOfWidthIncrements[keysOfWidthIncrements.index(tmpOrd)] + + + for tmpRowCur in range(c_startRow, c_endRow + 1): + for tmpColCur in range(c_startCol, c_endCol +1): + #print (tmpRowCur, tmpColCur) + r1,g1,b1,a1 = pixReloaded[tmpColCur, tmpRowCur] # Index col first, row second for image pixel array. TODO asdf this pix has been modified. All pixels would be transparent? - load image again? + if(a1 == 0): +# print "with alpha 8bit:", (r1, g1, b1, a1) + #make completely transparent - write 0x8000 + tmpPixelColorRGB555ToWrite = pack('H', 0x8000) + targetFontFile.write(tmpPixelColorRGB555ToWrite) # unsigned short - 2 bytes + else: # alpha should be 255 here really. + #print "8bit:", (r1, g1, b1) + tmp5bitR1 = (r1 >> 3) & 0x1f + tmp5bitG1 = (g1 >> 3) & 0x1f + tmp5bitB1 = (b1 >> 3) & 0x1f + #print "5bit:", (tmp5bitR1, tmp5bitG1, tmp5bitB1) + tmpPixelColorConvertedToRGB555 = (tmp5bitR1 << 10) | (tmp5bitG1 << 5) | (tmp5bitB1) + #print "16bit:", tmpPixelColorConvertedToRGB555 + tmpPixelColorRGB555ToWrite = pack('H', tmpPixelColorConvertedToRGB555) + targetFontFile.write(tmpPixelColorRGB555ToWrite) # unsigned short - 2 bytes + if (tmpColCur == c_endCol and explicitWidthIncrementVal > 0): + for tmpExtraColCur in range (0, explicitWidthIncrementVal): + #make completely transparent - write 0x8000 + tmpPixelColorRGB555ToWrite = pack('H', 0x8000) + targetFontFile.write(tmpPixelColorRGB555ToWrite) # unsigned short - 2 bytes + kIncIndx = kIncIndx + 1 # finally increase the kIncIndx for next glyph + + # + # End rest of the font characters image segments + # + targetFontFile.close() + # + # Re -open and write the image segment + # + targetFontFile = None + try: + targetFontFile = open(self.targetFONFilename, 'r+b') + except: + errorFound = True + if not errorFound: + targetFontFile.seek(0x0C) # position to write imageSegmentSize + tmpImageSegmentToWrite = pack('I', lastImageSegmentOffset) + targetFontFile.write(tmpImageSegmentToWrite) + targetFontFile.close() + + else: ## if (self.lettersFound ) <= 0 + errMsg = "No letters were found in input png!" + print errMsg + retVal = -2 + return (retVal, errMsg, origGameFontSizeEqBaseLine, totalFontLetters, importedNumOfLetters) + + def extractFonFilesFromMix(self): + """ Generate PNG files out of FON files stores in a MIX resource + """ + print "Checking in %s for MIX files to extract FON's from" % (self.inputFonMixPath) + inputMIXFilesFound = [] + # breaking after first for loop yields only the top directory files, which is what we want + for (dirpath, dirnames, filenames) in walk(self.inputFonMixPath): + for filename in filenames: + for mixFileName in supportedMIXInputFiles: + if filename.upper() == mixFileName: + inputMIXFilesFound.append(mixFileName) + break + for tmpMIXfileName in inputMIXFilesFound: + print "Found MIX: %s" % ('"' + self.inputFonMixPath + tmpMIXfileName + '"') + errorFound = False + inMIXFile = None + # + try: + inMIXFile = open(os.path.join(self.inputFonMixPath,tmpMIXfileName), 'rb') + except: + errorFound = True + print "Unexpected error:", sys.exc_info()[0] + raise + if not errorFound: + totalFONs = 0 + tmpBuff = inMIXFile.read(2) + # H: unsigned short (2 bytes) followed by I: unsigned int (4 bytes) + mixFileEntriesNumTuple = struct.unpack('H', tmpBuff) + numOfEntriesToExtract = mixFileEntriesNumTuple[0] + tmpBuff = inMIXFile.read(4) + mixFileDataSegmentSizeTuple = struct.unpack('I', tmpBuff) + allMixFileSize = mixFileDataSegmentSizeTuple[0] + inMIXFile.seek(0, 2) # go to file end + allActualBytesInMixFile = inMIXFile.tell() + inMIXFile.seek(6, 0) # go to start of table of MIX file entries (right after the 6 bytes header) + # 2 + 4 = 6 bytes short MIX header + # 12 bytes per MIX entry in entries table + # quick size validation + print "Entries: %d, data segment %d bytes" % (numOfEntriesToExtract, allMixFileSize) + if allActualBytesInMixFile != 2 + 4 + 12 * numOfEntriesToExtract + allMixFileSize: + print "Error: MIX file size mismatch with reported size in header for %s!" % (tmpMIXfileName) + else: + # + # 12 bytes per entry + # 4 bytes: ID + # 4 bytes: Offset in data segment + # 4 bytes: Size of data + # + for i in range(0, numOfEntriesToExtract): + foundFONFile = False + currFonFileName = 'UNKNOWN.FON' + inMIXFile.seek(2 + 4 + 12*i) + tmpBuff = inMIXFile.read(4) + tmpRdTuple = struct.unpack('I', tmpBuff) + idOfMIXEntry = tmpRdTuple[0] + tmpBuff = inMIXFile.read(4) + tmpRdTuple = struct.unpack('I', tmpBuff) + offsetOfMIXEntry = tmpRdTuple[0] + tmpBuff = inMIXFile.read(4) + tmpRdTuple = struct.unpack('I', tmpBuff) + sizeOfMIXEntry = tmpRdTuple[0] + + for suppFONFileName in supportedExportedFONFiles: + if(idOfMIXEntry == calculateFoldHash(suppFONFileName)): + foundFONFile = True + currFonFileName = suppFONFileName + break + + if (foundFONFile == True): + print "Entry Name: %s, Entry ID: %s, offset %s, data segment %s bytes" % (currFonFileName, ''.join('{:08X}'.format(idOfMIXEntry)), ''.join('{:08X}'.format(offsetOfMIXEntry)),''.join('{:08X}'.format(sizeOfMIXEntry))) + # + # IF FON FILE: + # put file in FON object + # + # + inMIXFile.seek(2 + 4 + 12*numOfEntriesToExtract + offsetOfMIXEntry) + if(offsetOfMIXEntry + sizeOfMIXEntry > allMixFileSize): + print "Error: FON file size mismatch with reported size in entry header!" + else: + fonFileBuffer = inMIXFile.read(sizeOfMIXEntry) + if (len(fonFileBuffer) == sizeOfMIXEntry): + # load FON file + thisFonFile = fonFile() + if (thisFonFile.loadFonFile(fonFileBuffer, allMixFileSize, currFonFileName)): + print "FON file loaded" + thisFonFile.outputFonToPNG() + totalFONs = totalFONs + 1 + else: + print "Error while LOADING FON file!" + else: + print "Error while reading FON file %s into mem buffer" % (''.join('{:08X}'.format(idOfMIXEntry))) + + inMIXFile.close() + print "Total FONs: %d " % (totalFONs) + return + +# +# +# ######################## +# main +# +# ######################### +# +if __name__ == '__main__': +# main() + invalidSyntax = False + extractFonMode = False + + TMPSpecialGlyphMode = True + TMPAutoTabCalculation = True + + TMPOverrideEncodingFilePath = "" + TMPinputPathForMixFiles = "" + TMPimageRowFilePNG = "" + TMPTargetFONfilename = "" + TMPminSpaceBetweenLettersInRowLeftToLeft = 0 + TMPminSpaceBetweenLettersInColumnTopToTop = 0 + TMPkerningForFirstDummyFontLetter = 0 + TMPYOffsToApplyToAllGlyphsExceptFirstSpecialGamma = 0 + TMPSpaceWidthInPixels = 10 +# TMPdeductKerningPixels = 0 + TMPcustomBaseLineOffset = 0 + +# print "Len of sysargv = %s" % (len(sys.argv)) + if len(sys.argv) == 2: + if(sys.argv[1] == '--help'or sys.argv[1] == '-h'): + print "%s %s supports Blade Runner (English version, CD edition)." % (app_name_spaced, app_version) + print "Created by Praetorian of the classic adventures in Greek team." + print "Always keep backups!" + print "--------------------" + print "Preparatory steps:" + print "1. Put overrideEncoding.txt file in the same folder with this tool. (Recommended, but not obligatory step)" + print "--------------------" + print "Valid syntax A - export game fonts:" + print "%s -ip [folderpath_for_MIX_Files]\n" % (app_name) + print "Valid syntax B - create subtitle font:" + print "%s -im [image_Row_PNG_Filename] -om [output_FON_filename] -pxLL [minSpaceBetweenLettersInRowLeftToLeft] -pxTT [minSpaceBetweenLettersInColumnTopToTop] -pxKn [kerningForFirstDummyFontLetter] -pxWS [whiteSpaceWidthInPixels]\n" % (app_name) # deductKerningPixels" + print "The -ip switch has an argument that is the path for the input (MIX) files folder (can be the same as the Blade Runner installation folder)." + print "The -oe switch has an argument that is the input overrideEncoding file to use for the particular font creation." + print "The -im switch has an argument that is the input PNG image with a row of the font glyphs spaced apart." + print "The -om switch has an argument that is the output FON filename." + print "The -pxLL switch has an integer argument that specifies the minimum number of pixels between the left side of a glyph and the left side of the next glyph to its right in the line-row PNG." + print "The -pxTT switch has an integer argument that specifies the minimum number of pixels between the top side of a glyph and the top side of the glyph below (if there's a second row) in the row PNG. If there is only one row, this argument still should be set (as if there was another row) to define where the parser should stop checking for the limits of a glyph vertically." + print "The -pxKn switch has an integer argument that sets kerning for the first dummy font glyph." + print "The -pxYo switch has an integer argument that sets an offset to be added to all detected y offsets for the glyphs (except the special first one)." + print "The -pxWS switch has an integer argument that sets the white space width in pixels for this particular font." + print "The --noSpecialGlyphs switch removes consideration for special glyphs that exist out of their proper ascii order." + print "The --noAutoTabCalculation switch removes the detection of tab spacing between letters (use this switch if you didn't create the PNG row file using a tab spaced list of glyphs)." + print "--------------------" + print "Thank you for using this app." + print "Please provide any feedback to: %s " % (company_email) + sys.exit() + elif(sys.argv[1] == '--version' or sys.argv[1] == '-v'): + print "%s %s supports Blade Runner (English version, CD edition)." % (app_name_spaced, app_version) + print "Please provide any feedback to: %s " % (company_email) + sys.exit() + else: + invalidSyntax = True + elif len(sys.argv) > 2: + for i in range(1, len(sys.argv)): + if( i < (len(sys.argv) - 1) and sys.argv[i][:1] == '-' and sys.argv[i+1][:1] != '-'): + if (sys.argv[i] == '-ip'): + TMPinputPathForMixFiles = sys.argv[i+1] + extractFonMode = True + print "Original FON file extraction mode enabled." + elif (sys.argv[i] == '-oe'): + TMPOverrideEncodingFilePath = sys.argv[i+1] + elif (sys.argv[i] == '-im'): + TMPimageRowFilePNG = sys.argv[i+1] + elif (sys.argv[i] == '-om'): + TMPTargetFONfilename = sys.argv[i+1] + elif (sys.argv[i] == '-pxLL'): + TMPminSpaceBetweenLettersInRowLeftToLeft = int(sys.argv[i+1]) + elif (sys.argv[i] == '-pxTT'): + TMPminSpaceBetweenLettersInColumnTopToTop = int(sys.argv[i+1]) + elif (sys.argv[i] == '-pxKn'): + TMPkerningForFirstDummyFontLetter = int(sys.argv[i+1]) + elif (sys.argv[i] == '-pxYo'): + TMPYOffsToApplyToAllGlyphsExceptFirstSpecialGamma = int(sys.argv[i+1]) + elif (sys.argv[i] == '-pxWS'): + TMPSpaceWidthInPixels = int(sys.argv[i+1]) + elif sys.argv[i] == '--noSpecialGlyphs': + print "No special out-of-order glyphs mode enabled." + TMPSpecialGlyphMode = False + elif sys.argv[i] == '--noAutoTabCalculation': + print "No automatic tab calculation between glyphs." + TMPAutoTabCalculation = False + + if (extractFonMode == False) and (not TMPTargetFONfilename or not TMPimageRowFilePNG or TMPminSpaceBetweenLettersInRowLeftToLeft <= 0 or TMPminSpaceBetweenLettersInColumnTopToTop <= 0 or TMPkerningForFirstDummyFontLetter <= 0 or TMPSpaceWidthInPixels <= 0) : # this argument is mandatory + invalidSyntax = True + + if (extractFonMode == True) and ( (TMPinputPathForMixFiles == '') or not TMPOverrideEncodingFilePath ): + invalidSyntax = True + else: + invalidSyntax = True + + + if (invalidSyntax == False): + #myGrabInstance = grabberFromPNG('windows-1253') #, grabberFromPNG.BR_GameID) + myGrabInstance = grabberFromPNG() + myGrabInstance.setInputPathForFonMix(TMPinputPathForMixFiles) + myGrabInstance.setImageRowFilePNG(TMPimageRowFilePNG) + myGrabInstance.setTargetFONFilename(TMPTargetFONfilename) + myGrabInstance.setMinSpaceBetweenLettersInRowLeftToLeft(TMPminSpaceBetweenLettersInRowLeftToLeft) + myGrabInstance.setMinSpaceBetweenLettersInColumnTopToTop(TMPminSpaceBetweenLettersInColumnTopToTop) + myGrabInstance.setKerningForFirstDummyFontLetter(TMPkerningForFirstDummyFontLetter) + myGrabInstance.setYOffsetForAllGlyphsExceptFirstSpecialGamma(TMPYOffsToApplyToAllGlyphsExceptFirstSpecialGamma) + myGrabInstance.setSpaceWidthInPixels(TMPSpaceWidthInPixels) + myGrabInstance.setSpecialGlyphMode(TMPSpecialGlyphMode) + myGrabInstance.setAutoTabCalculation(TMPAutoTabCalculation) + myGrabInstance.setOverrideEncodingPath(TMPOverrideEncodingFilePath) + myGrabInstance.initOverrideEncoding() +# myGrabInstance.setDeductKerningPixels(TMPdeductKerningPixels) + if extractFonMode: + myGrabInstance.extractFonFilesFromMix() + else: + myGrabInstance.generateModFiles(TMPcustomBaseLineOffset) + else: + invalidSyntax = True + + if invalidSyntax == True: + print "Invalid syntax\n Try: \n %s -op [folderpath_for_extracted_wav_Files] \n %s --help for more info \n %s --version for version info " % (app_name, app_name, app_name) + tmpi = 0 + for tmpArg in sys.argv: + if tmpi==0: #skip first argument + tmpi+=1 + continue + print "\nArgument: %s" % (tmpArg) + tmpi+=1 +else: + #debug + #print 'font grabber imported from another module' + pass diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/KIA6PT.FON-Ext012TranspZeroThresh0002.png b/devtools/blade_runner/subtitles/fontCreator/samples/KIA6PT.FON-Ext012TranspZeroThresh0002.png Binary files differnew file mode 100644 index 0000000000..85cab20f36 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/KIA6PT.FON-Ext012TranspZeroThresh0002.png diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersKIASimple.txt b/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersKIASimple.txt new file mode 100644 index 0000000000..f7da07f2f0 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersKIASimple.txt @@ -0,0 +1 @@ +python2.7 grabberFromPNG17BR.py -im ./samples/KIA6PT.FON-Ext012TranspZeroThresh0002.png -oe ./samples/overrideEncodingKIA6PT.txt -om KIA6PT.FON -pxLL 14 -pxTT 15 -pxKn 1 -pxWS 4 -pxYo 1 --noAutoTabCalculation
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersSUBTLS.txt b/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersSUBTLS.txt new file mode 100644 index 0000000000..128351f580 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersSUBTLS.txt @@ -0,0 +1 @@ +python2.7 grabberFromPNG17BR.py -im ./samples/Tahoma_18Shdw-G3NewMrgd.png -oe ./samples/overrideEncodingSUBLTS.txt -om SUBTLS_E.FON -pxLL 42 -pxTT 30 -pxKn 1 -pxWS 7
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersTahoma18Simple.txt b/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersTahoma18Simple.txt new file mode 100644 index 0000000000..ed2dea15d7 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersTahoma18Simple.txt @@ -0,0 +1 @@ +python2.7 grabberFromPNG17BR.py -im ./samples/Tahoma_18ShdwSimpleN4TransOnBlack.png -oe ./samples/overrideEncodingTahoma18.txt -om TAHOMA18.FON -pxLL 42 -pxTT 30 -pxKn 1 -pxWS 5 -pxYo 3
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersTahoma24Simple.txt b/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersTahoma24Simple.txt new file mode 100644 index 0000000000..ad6f9de4b4 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/SampleCMDParametersTahoma24Simple.txt @@ -0,0 +1 @@ +python2.7 grabberFromPNG17BR.py -im ./samples/Tahoma_24ShdwSimpleN3BlackMrgd.png -oe ./samples/overrideEncodingTahoma24.txt -om TAHOMA24.FON -pxLL 45 -pxTT 40 -pxKn 1 -pxWS 8 -pxYo 2
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_18Shdw-G3NewMrgd.png b/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_18Shdw-G3NewMrgd.png Binary files differnew file mode 100644 index 0000000000..bed9020750 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_18Shdw-G3NewMrgd.png diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_18Shdw-G3NewMrgd.xcf b/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_18Shdw-G3NewMrgd.xcf Binary files differnew file mode 100644 index 0000000000..c0949da6ce --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_18Shdw-G3NewMrgd.xcf diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_18ShdwSimpleN4TransOnBlack.png b/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_18ShdwSimpleN4TransOnBlack.png Binary files differnew file mode 100644 index 0000000000..2ecc871fe4 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_18ShdwSimpleN4TransOnBlack.png diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_24ShdwSimpleN3BlackMrgd.png b/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_24ShdwSimpleN3BlackMrgd.png Binary files differnew file mode 100644 index 0000000000..d0c930b7a3 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/Tahoma_24ShdwSimpleN3BlackMrgd.png diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/actualFontsNeededToCopyToGimpTabbed.txt b/devtools/blade_runner/subtitles/fontCreator/samples/actualFontsNeededToCopyToGimpTabbed.txt new file mode 100644 index 0000000000..dbf506bf58 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/actualFontsNeededToCopyToGimpTabbed.txt @@ -0,0 +1 @@ +! ! " # $ % & ' ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~ … í Ά ñ â é Έ Ή Ί Ό Ύ Ώ ΐ Α Β Γ Δ Ε Ζ Η Θ Ι Κ Λ Μ Ν Ξ Ο Π Ρ Σ Τ Υ Φ Χ Ψ Ω Ϊ Ϋ ά έ ή ί ΰ α β γ δ ε ζ η θ ι κ λ μ ν ξ ο π ρ ς σ τ υ φ χ ψ ω ϊ ϋ ό ύ ώ
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/overrideEncodingKIA6PT.txt b/devtools/blade_runner/subtitles/fontCreator/samples/overrideEncodingKIA6PT.txt new file mode 100644 index 0000000000..24fb8855cc --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/overrideEncodingKIA6PT.txt @@ -0,0 +1,7 @@ +targetEncoding=windows-1253 +asciiCharList=!!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~–ƒΆΈΉΊΌΎΏΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώ +explicitKerningList=!:1,;:1,`:1,{:1,|:1,}:1 +explicitWidthIncrement=i:0,j:1,l:1 +originalFontName=KIA6PT +specialOutOfOrderGlyphsUTF8ToAsciiTargetEncoding=é:ƒ,ü:– + diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/overrideEncodingSUBLTS.txt b/devtools/blade_runner/subtitles/fontCreator/samples/overrideEncodingSUBLTS.txt new file mode 100644 index 0000000000..a4d403f874 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/overrideEncodingSUBLTS.txt @@ -0,0 +1,7 @@ +targetEncoding=windows-1253 +asciiCharList=!!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~…Ά£¥¦§ΈΉΊΌΎΏΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώ +explicitKerningList=i:-1 +explicitWidthIncrement=i:0,j:1,l:1 +originalFontName=SUBLTS +specialOutOfOrderGlyphsUTF8ToAsciiTargetEncoding=í:Ά,ñ:¥,â:¦,é:§,Ά:£ + diff --git a/devtools/blade_runner/subtitles/fontCreator/samples/overrideEncodingTahoma24.txt b/devtools/blade_runner/subtitles/fontCreator/samples/overrideEncodingTahoma24.txt new file mode 100644 index 0000000000..9cb2ca0e41 --- /dev/null +++ b/devtools/blade_runner/subtitles/fontCreator/samples/overrideEncodingTahoma24.txt @@ -0,0 +1,7 @@ +targetEncoding=windows-1253 +asciiCharList=!!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~–ƒΆΈΉΊΌΎΏΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώ +explicitKerningList=i:-1 +explicitWidthIncrement=i:0,j:1,l:1 +originalFontName=TAHOMA +specialOutOfOrderGlyphsUTF8ToAsciiTargetEncoding=é:ƒ,ü:– + diff --git a/devtools/blade_runner/subtitles/mixResourceCreator/packBladeRunnerMIXFromPCTLKXLS-04.py b/devtools/blade_runner/subtitles/mixResourceCreator/packBladeRunnerMIXFromPCTLKXLS-04.py new file mode 100644 index 0000000000..2ebb5e9fe3 --- /dev/null +++ b/devtools/blade_runner/subtitles/mixResourceCreator/packBladeRunnerMIXFromPCTLKXLS-04.py @@ -0,0 +1,725 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +# +# Created by Praetorian (ShadowNate) for Classic Adventures in Greek +# classic.adventures.in.greek@gmail.com +# Works with Excel version outSpeech-15-06-2018-1856-TranslatingComms-080.xls and above +# +# TODO Support at least one translation too (ie Greek) +# Print a warning if packing a TRE without the corresponding font(s) -- only a warning though +# +import os, sys, shutil +import ctypes +import csv +import os.path +import xlrd +from xlrd import * +# for pack +from struct import * +import re + +company_email = "classic.adventures.in.greek@gmail.com" +app_version = "0.60" +app_name = "packBladeRunnerMIXFromPCTLKXLS" +app_name_spaced = "Get a TRE file from spoken in-game quotes" +numOfSpokenQuotes = 0 + +defaultSubtitlesFontName = 'SUBTLS_E.FON' + +origEncoding = 'windows-1252' +defaultTargetEncoding = 'windows-1252' +defaultTargetEncodingUnicode = unicode(defaultTargetEncoding, 'utf-8') +targetEncoding = '' +targetEncodingUnicode = '' + +configureFontsTranslationTextFile = u'configureFontsTranslation.txt' +relPath = u'.' +configureFontsTranslationTextFileRelPath = os.path.join(relPath, configureFontsTranslationTextFile) + + +# DONE ADD ALL SHEETS NEEDED FROM THE XLS +# all dialogue sheets get the SUBTLS_E.FON for translation to TRE +# - TODO maybe merge this with TAHOMA18.FON eventually +supportedDialogueSheets = ['INGQUO_E.TRE', 'WSTLGO_E.VQA', 'BRLOGO_E.VQA', 'INTRO_E.VQA', 'MW_A_E.VQA', 'MW_B01_E.VQA', 'MW_B02_E.VQA', 'MW_B03_E.VQA', 'MW_B04_E.VQA', 'MW_B05_E.VQA', 'INTRGT_E.VQA', 'MW_D_E.VQA', 'MW_C01_E.VQA', 'MW_C02_E.VQA', 'MW_C03_E.VQA', 'END04A_E.VQA', 'END04B_E.VQA', 'END04C_E.VQA', 'END06_E.VQA', 'END01A_E.VQA', 'END01B_E.VQA', 'END01C_E.VQA', 'END01D_E.VQA', 'END01E_E.VQA', 'END01F_E.VQA', 'END03_E.VQA'] +# +# Each TRE sheet gets a specific font to handle their translation to TRE +# TAHOMA means both TAHOMA (their translation should be identical (although in the original they have minor differences but they don't affect anything) +# We use a single naming for TAHOMA here because both TAHOMA18 and TAHOMA24 are used for ENDCRED.TRE +# The TRE files that are identically named to the originals are supposed to override them (needs ScummVM compatible functionality for that) + +supportedTranslationSheets = [('OPTIONS.TRE', 'KIA6PT'), ('DLGMENU.TRE', 'KIA6PT'), ('SCORERS.TRE', 'TAHOMA'), ('VK.TRE', 'KIA6PT'), ('CLUES.TRE', 'KIA6PT'), ('CRIMES.TRE', 'KIA6PT'), ('ACTORS.TRE', 'KIA6PT'), ('HELP.TRE', 'KIA6PT'), ('AUTOSAVE.TRE', 'KIA6PT'), ('ERRORMSG.TRE', 'KIA6PT'), ('SPINDEST.TRE', 'KIA6PT'), ('KIA.TRE', 'KIA6PT'), ('KIACRED.TRE', 'KIA6PT'), ('CLUETYPE.TRE', 'KIA6PT'), ('ENDCRED.TRE', 'TAHOMA'), ('POGO.TRE', 'KIA6PT')] +# The FON files that are identically named to the originals are supposed to override them (needs ScummVM compatible functionality for that) +supportedOtherFilesForMix = [defaultSubtitlesFontName, 'KIA6PT.FON', 'TAHOMA18.FON', 'TAHOMA24.FON'] # , '10PT.FON'] # we don't deal with 10PT.FON since it's not used -- TODO verify this. + + +tableOfStringIds = [] +tableOfStringOffsets = [] +tableOfStringEntries = [] + +# this list is used in order to replace the actual indices of characters with delegate font indices (ASCII indexes of the target code-page) which have been used during the font creation (or exist in in the internal TAHOMA font) +# contains tuples of two values. First value is actual Utf char, the second is a replacement ASCII char +listOfFontNamesToOutOfOrderGlyphs = [] +arrangedListOfFontNamesToOutOfOrderGlyphs = [] + +actorPropertyEntries = [] +actorPropertyEntriesWasInit = False + +def initOverrideEncoding(): + global targetEncoding + + configureTranslationFailed = True + try: + if os.access(configureFontsTranslationTextFileRelPath, os.F_OK): + conFontsTranslationFile = open(configureFontsTranslationTextFileRelPath, 'r') + linesLst = conFontsTranslationFile.readlines() + conFontsTranslationFile.close() + if linesLst is None or len(linesLst) == 0: + configureTranslationFailed = True + else: + print "Font Translation Configuration Info: " + involvedTokensLst =[] + for readEncodLine in linesLst: + tmplineTokens = re.findall("[^\t\n]+",readEncodLine ) + for x in tmplineTokens: + involvedTokensLst.append(x) + + for tokenNameKeyPair in involvedTokensLst: + nameKeyTupl = tokenNameKeyPair.split('=', 1) + try: + if len(nameKeyTupl) == 2 and nameKeyTupl[0] == 'targetEncoding' and nameKeyTupl[1] is not None and nameKeyTupl[1] != '-' and nameKeyTupl[1] != '': + targetEncodingUnicode = unicode(nameKeyTupl[1], 'utf-8') + targetEncoding = unicode.encode("%s" % targetEncodingUnicode, origEncoding) + elif len(nameKeyTupl) == 2 and nameKeyTupl[0] == 'fontNameAndOutOfOrderGlyphs' and nameKeyTupl[1] is not None and nameKeyTupl[1] != '': + # split at hash tag first + tmpListOfOutOfOrderGlyphs = [] + del(tmpListOfOutOfOrderGlyphs[:]) + fontNameAndOOOGlyphsTuple = nameKeyTupl[1].split('#', 1) + if (len (fontNameAndOOOGlyphsTuple) == 2 and fontNameAndOOOGlyphsTuple[0] != '' and fontNameAndOOOGlyphsTuple[1] is not None and fontNameAndOOOGlyphsTuple[1] != ''): + tmpFontName = fontNameAndOOOGlyphsTuple[0] + # split at comma, then split at ':' and store tuples of character + explicitOutOfOrderGlyphsTokenUnicode = unicode(fontNameAndOOOGlyphsTuple[1], 'utf-8') # unicode(fontNameAndOOOGlyphsTuple[1], 'utf-8') + #explicitOutOfOrderGlyphsTokenStr = unicode.encode("%s" % explicitOutOfOrderGlyphsTokenUnicode, targetEncoding) + #explicitOutOfOrderGlyphsTokenStr = explicitOutOfOrderGlyphsTokenUnicode.decode(targetEncoding) # unicode.encode("%s" % explicitOutOfOrderGlyphsTokenUnicode, 'utf-8') + tokensOfOutOfOrderGlyphsStrList = explicitOutOfOrderGlyphsTokenUnicode.split(',') + for tokenX in tokensOfOutOfOrderGlyphsStrList: + tokensOfTupleList = tokenX.split(':') + tmpListOfOutOfOrderGlyphs.append( (unichr(ord(tokensOfTupleList[0])), unichr(ord(tokensOfTupleList[1]))) ) + + if tmpFontName not in [x[0] for x in listOfFontNamesToOutOfOrderGlyphs]: + listOfFontNamesToOutOfOrderGlyphs.append( ( tmpFontName, tmpListOfOutOfOrderGlyphs) ) + + else: + configureTranslationFailed = True + break + except: + configureTranslationFailed = True + raise + + if not (targetEncoding is None or targetEncoding == ''): + configureTranslationFailed = False + + except: + print "Error while trying to access file for encoding info: %s" % (configureFontsTranslationTextFileRelPath) + raise + configureTranslationFailed = True + + if configureTranslationFailed == True: +# targetEncoding = defaultTargetEncoding + print "Error! Could not find proper override encoding info in: %s" % (configureFontsTranslationTextFileRelPath) + sys.exit() # terminate if override Failed (Blade Runner) + # + # TODO ASDF fix this!!! + # + if(len(listOfFontNamesToOutOfOrderGlyphs) == 0): + tmpFontType = defaultSubtitlesFontName[:-4] # remove the .FON extensionFromTheName + print "Empty list for out of order glyphs. Assuming default out of order glyphs and only for the %s font" % (tmpFontType) + tmplistOfOutOfOrderGlyphs = [] + tmplistOfOutOfOrderGlyphs.append((u'\xed', u'\u0386')) # spanish i (si) + tmplistOfOutOfOrderGlyphs.append((u'\xf1', u'\xa5')) # spanish n (senor) + tmplistOfOutOfOrderGlyphs.append((u'\xe2', u'\xa6')) # a for (liver) pate + tmplistOfOutOfOrderGlyphs.append((u'\xe9', u'\xa7')) # e for (liver) pate + listOfFontNamesToOutOfOrderGlyphs.append( (tmpFontType, tmplistOfOutOfOrderGlyphs)) + print "Explicit Out Of Order Glyphs List: " , listOfFontNamesToOutOfOrderGlyphs + # arrange list properly: + # check if the list contains same item as key and value (in different pairs) + # if such case then the pair with the key should preceed the pair with the value matched, + # to avoid replacing instances of a special character (key) with a delegate (value) that will be later replaced again due to the second pair + # + for (itFontName, itOOOGlyphList) in listOfFontNamesToOutOfOrderGlyphs: + while (True): + foundMatchingPairs = False + for glyphDelegItA in itOOOGlyphList: + for glyphDelegItB in itOOOGlyphList: + if (glyphDelegItA[1] == glyphDelegItB[0] and itOOOGlyphList.index(glyphDelegItA) < itOOOGlyphList.index(glyphDelegItB)): + # swap + itamA, itamB = itOOOGlyphList.index(glyphDelegItA), itOOOGlyphList.index(glyphDelegItB) + itOOOGlyphList[itamB], itOOOGlyphList[itamA] = itOOOGlyphList[itamA], itOOOGlyphList[itamB] + foundMatchingPairs = True + break + if (foundMatchingPairs == True): + break + if(foundMatchingPairs == False): + break # the whole while loop + arrangedListOfFontNamesToOutOfOrderGlyphs.append( ( itFontName, itOOOGlyphList)) + print "Arranged Glyphs Delegates List: " , arrangedListOfFontNamesToOutOfOrderGlyphs + return + +# +# Fill the actorPropertyEntries table +def initActorPropertyEntries(): + global actorPropertyEntriesWasInit + global actorPropertyEntries + firstLine = True +# print "opening actornames" + with open("./actornames.txt") as tsv: + for line in csv.reader(tsv, dialect="excel-tab"): + #skip first line header + if firstLine == True: +# print "skippingHeader" + firstLine = False + else: + actorPropertyEntries.append(line) + actorPropertyEntriesWasInit = True + tsv.close() + +def getActorShortNameById(lookupActorId): + global actorPropertyEntriesWasInit + global actorPropertyEntries + if not actorPropertyEntriesWasInit: + return '' + else: + for actorEntryTmp in actorPropertyEntries: + if int(actorEntryTmp[0]) == int(lookupActorId): + return actorEntryTmp[1] + return '' + + +def getActorFullNameById(lookupActorId): + global actorPropertyEntriesWasInit + global actorPropertyEntries + if not actorPropertyEntriesWasInit: + return '' + else: + for actorEntryTmp in actorPropertyEntries: + if int(actorEntryTmp[0]) == int(lookupActorId): + return actorEntryTmp[2] + return '' + +def getActorIdByShortName(lookupActorShortName): + global actorPropertyEntriesWasInit + global actorPropertyEntries + if not actorPropertyEntriesWasInit: + return '' + else: + for actorEntryTmp in actorPropertyEntries: + if actorEntryTmp[1] == lookupActorShortName: + return actorEntryTmp[0].zfill(2) + return '' + +# +# +# FOR MIX FILE +# + +# strFileName should be the full file name (including extension) +def calculateFoldHash(strFileName): + i = 0 + hash = 0 + strParam = strFileName.upper() + lenFileName = len(strParam); + while i < lenFileName and i < 12: + groupSum = 0 + # work in groups of 4 bytes + for j in range(0, 4): + # LSB first, so the four letters in the string are re-arranged (first letter goes to lower place) + groupSum >>= 8; + if (i < lenFileName): + groupSum |= (ord(strParam[i]) << 24) + i += 1 + else: # if i >= lenFileName but still haven't completed the four byte loop add 0s + groupSum |= 0 + hash = ((hash << 1) | ((hash >> 31) & 1)) + groupSum + hash &= 0xFFFFFFFF # mask here! + print (strParam +': ' +''.join('{:08X}'.format(hash))) + return hash + +# +# aux - sort by first object in list of tuples +def getSortMixFilesKey(item): + keyTmp = item[0] & 0xFFFFFFFF + + signedKeyTmp = ctypes.c_long(keyTmp).value + return signedKeyTmp +# +def outputMIX(): + # output file should be SUBTITLES.MIX + # checking with known hashes to verify calculateFoldHash + #calculateFoldHash('AR01-MIN.SET') + #calculateFoldHash('AR02-MIN.SET') + #calculateFoldHash('CLOVDIES.AUD') + #calculateFoldHash('INTRO.VQA') + + errorFound = False + outMIXFile = None + try: + outMIXFile = open("./SUBTITLES.MIX", 'wb') + except: + errorFound = True + if not errorFound: + # Write header + # 2 bytes: number of entries (NumFiles) + # TODO 4 bytes: size of data segment + # 12 * NumFiles bytes: Entry descriptors table + # 4 bytes: ID (hash) + # 4 bytes: Byte offset in Data Segment + # 4 bytes: Byte length of entry data + # TODO *Data Segment* - contains the file data. Offset from Entry Descriptors does not include header segment byte length. + # Note that the offsets are relative to the start of the body so to find the + # actual offset in the MIX you have to add the size of the header which is + # (6 + (12 * NumFiles)) + + # + # ID column should in ascending order in MIX FILES (the engine uses binary sort to search for files) + # so order the files based on ID hash + # Create a list of 3-item tuples, first item is id, second item is filename + # Then sort the list + # Then write to entry table + # + # Also filenames should be 8 characters at most and 4 more for extension to conform with specs + # ^^ this is done manually by making sure the filenames in the sheets of the excel as compliant + # Based on observations from STARTUP.MIX: + # 1) the hash ids can overflow and so lower numbers seem to appear down in the index table entries list + # -- So we sort hash but we first tranlste the unsigned key to signed with ctypes + # 2) the offsets are not necessarily sorted, meaning that the first entry in the index table won't necessarily have the 0x00000000 offset + i = 0 + mixFileEntries = [] + totalFilesDataSize = 0 + currOffsetForDataSegment = 0 # we start after header and table of index entries, from 0, (but this means that when reading the offset we need to add 6 + numOfFiles * 12). This does not concern us though. + for sheetDialogueName in supportedDialogueSheets: + sheetDialogueNameTRE = sheetDialogueName[:-4] + '.TRE' + if os.path.isfile('./' + sheetDialogueNameTRE): + entryID = calculateFoldHash(sheetDialogueNameTRE) + mixEntryfileSizeBytes = os.path.getsize('./' + sheetDialogueNameTRE) + mixFileEntries.append((entryID, sheetDialogueNameTRE, mixEntryfileSizeBytes)) + totalFilesDataSize += mixEntryfileSizeBytes + + for translatedTREFileName in [ x[0] for x in supportedTranslationSheets] : + if os.path.isfile('./' + translatedTREFileName): + entryID = calculateFoldHash(translatedTREFileName) + mixEntryfileSizeBytes = os.path.getsize('./' + translatedTREFileName) + mixFileEntries.append((entryID, translatedTREFileName, mixEntryfileSizeBytes)) + totalFilesDataSize += mixEntryfileSizeBytes + + for otherFileName in supportedOtherFilesForMix: + if os.path.isfile('./' + otherFileName): + entryID = calculateFoldHash(otherFileName) + mixEntryfileSizeBytes = os.path.getsize('./' + otherFileName) + mixFileEntries.append((entryID, otherFileName, mixEntryfileSizeBytes)) + totalFilesDataSize += mixEntryfileSizeBytes + mixFileEntries.sort(key=getSortMixFilesKey) + # + # We write num of files here. After we verified they exist + # + numOfFiles = len(mixFileEntries) + numOfFilesToWrite = pack('h',numOfFiles) # short 2 bytes + outMIXFile.write(numOfFilesToWrite) + + # This is just the data segment (after the entries index table). Adds up all the file sizes here + totalFilesDataSizeToWrite = pack('I',totalFilesDataSize) # unsigned integer 4 bytes + outMIXFile.write(totalFilesDataSizeToWrite) + + print ("Sorted Entries based on EntryId") + for mixFileEntry in mixFileEntries: + print (''.join('{:08X}'.format(mixFileEntry[0])) + ': ' + mixFileEntry[1] + ' : ' + ''.join('{:08X}'.format(mixFileEntry[2]))) + entryID = mixFileEntry[0] & 0xFFFFFFFF + entryIDToWrite = pack('I',entryID) # unsigned integer 4 bytes + outMIXFile.write(entryIDToWrite) + entryOffset = currOffsetForDataSegment # offsets have base after header and table of index entries + entryOffsetToWrite = pack('I',entryOffset) # unsigned integer 4 bytes + outMIXFile.write(entryOffsetToWrite) + entryByteLength = mixFileEntry[2] # File size + entryByteLengthToWrite = pack('I',entryByteLength) # unsigned integer 4 bytes + outMIXFile.write(entryByteLengthToWrite) + currOffsetForDataSegment += entryByteLength + # Add data segments here + errorReadingFound = False + for mixFileEntry in mixFileEntries: + try: + inEntryMIXFile = open("./"+ mixFileEntry[1], 'rb') + except: + errorReadingFound = True + if not errorReadingFound: + outMIXFile.write(inEntryMIXFile.read()) + inEntryMIXFile.close() + else: + print ("Error while reading in ENTRY file") + break + + outMIXFile.close() + print "TOTAL RESOURCES IN MIX: %d" % (numOfFiles) + return +# +# END FOR MIX FILE +# +# + +#def inputXLS(filename) + #TODO extra pass once the quotes have been updated for weird unicode characters + #TODO some ' quotes appear as \u2019 and others appear normally as '. what's that about? + #DONE manually I've replaced all weird \u2019 single quotes with ''' + #the spanish n is \xf1 -> we put it at ascii value: \xA5 -- font index 0xA6 ? + #the spanish i is \xed -> we put it at ascii value: \xA2 -- font index 0xA3 ? + #pâté + # a actual ascii value is 0xE2 in codepage 1252 -- put it in ascii value 0xA6 (165) -- font index 0xA7 + # e actual ascii value is 0xE9 in codepage 1252 -- put it in ascii value 0xA7 (166) -- font index 0xA8 + #TODO what are other characters are special? + #TODO transition to ASCII chars to store in TRE file? + #DONE manually I've replaced all one-char '...' with three dots + # TODO actors TRE has 0x49 entries, (73 names), but table of ids has 73 entries BUT the offset table (first offset is calced + 0x04, so from end of the first 4 count bytes) has 74 entries. The last entry indexes the end of file (!) + # TODO all strings are NULL terminated in the TRE file! + +def translateQuoteToAsciiProper(cellObj, pSheetName): + newQuoteReplaceSpecials = cellObj.value.encode("utf-8") + #print ('Encoded to unicode: %s ' % (newQuoteReplaceSpecials)) + newQuoteReplaceSpecials = newQuoteReplaceSpecials.decode("utf-8") + + pertinentListOfOutOfOrderGlyphs = [] + #print pSheetName + #print supportedDialogueSheets + #print defaultSubtitlesFontName[:-4] + #print [x[0] for x in listOfFontNamesToOutOfOrderGlyphs] + if pSheetName in supportedDialogueSheets and defaultSubtitlesFontName[:-4] in [x[0] for x in listOfFontNamesToOutOfOrderGlyphs]: + for (tmpFontName, tmpOOOList) in listOfFontNamesToOutOfOrderGlyphs: + if tmpFontName == defaultSubtitlesFontName[:-4]: + pertinentListOfOutOfOrderGlyphs = tmpOOOList + break + elif pSheetName in [x[0] for x in supportedTranslationSheets]: + pertinentFontType = '' + #[treAndFontTypeTuple for treAndFontTypeTuple in supportedTranslationSheets if treAndFontTypeTuple[0] == pSheetName] + for (tmpSheetName, tmpFontType) in supportedTranslationSheets: + if tmpSheetName == pSheetName: + pertinentFontType = tmpFontType + break + for (tmpFontName, tmpOOOList) in listOfFontNamesToOutOfOrderGlyphs: + if tmpFontName == pertinentFontType: + pertinentListOfOutOfOrderGlyphs = tmpOOOList + break + + #newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u0386", u"\u00A3") + for repTuple in pertinentListOfOutOfOrderGlyphs: + newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(repTuple[0], repTuple[1]) + # WORKAROUND, we re-replace the spanish i delegate again here! +# newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u'\xa2', u'\u0386') # this is needed for spanish i because in utf-8 it's actually the u'\u0386' that's assigned to A tonomeno which is the delegate. +# newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u0386", u"\u00A3") +# #newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u0386", u"\u00A3") # greek alpha tonomeno -- TODO which character is this in the excel (utf value) ??? +# newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u00ed", u"\u00A2") # spanish i +# newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u00f1", u"\u00A5") # spanish n +# #newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u00A4", u"\u00A5") # spanish n +# newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u00e2", u"\u00A6") # a from pate -- todo this is not confirmed in-game font (but it is in our external font as of yet) +# newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u00e9", u"\u00A7") # e from pate -- todo this is not confirmed in-game font (but it is in our external font as of yet) + # other replacements. + newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u2019", u"\u0027") # right single quote + newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u2018", u"\u0027") # left single quote + newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u2026", u"\u002e\u002e\u002e") # three dots together (changes length) + newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u201D", u"\u0022") # right double quote + newQuoteReplaceSpecials = newQuoteReplaceSpecials.replace(u"\u201C", u"\u0022") # left double quote + # TODO? replace new line ??? with another char (maybe |)? + + #newQuoteReplaceSpecialsUnicode = unicode(newQuoteReplaceSpecials, 'utf-8') + #newQuoteReplaceSpecialsStr = unicode.encode("%s" % newQuoteReplaceSpecials, targetEncoding) + + #print type(newQuoteReplaceSpecials) # type is unicode + #print type(newQuoteReplaceSpecials.encode('utf-8')) # type is str +# print targetEncoding +# print newQuoteReplaceSpecials +# newQuoteReplaceSpecialsDec = newQuoteReplaceSpecials.decode(targetEncoding) + newQuoteReplaceSpecialsRetStr = '' + newQuoteReplaceSpecialsRetStr = newQuoteReplaceSpecials.encode(targetEncoding) +# try: +# newQuoteReplaceSpecialsRetStr = newQuoteReplaceSpecials.encode(targetEncoding) +# except: +# print "===============================================================================" +# print "===============================================================================" +# print "ERROR:" +# print newQuoteReplaceSpecials +# print newQuoteReplaceSpecials.encode(targetEncoding, errors='xmlcharrefreplace') +# print "===============================================================================" +# print "===============================================================================" +# newQuoteReplaceSpecialsRetStr = newQuoteReplaceSpecials.encode(targetEncoding, errors='xmlcharrefreplace') + return newQuoteReplaceSpecialsRetStr +# return newQuoteReplaceSpecialsEnStr + + +def inputXLS(filename): + global numOfSpokenQuotes + global tableOfStringIds + global tableOfStringOffsets + global tableOfStringEntries + # Open the workbook + xl_workbook = xlrd.open_workbook(filename, encoding_override="utf-8") + + + # List sheet names, and pull a sheet by name + # + # sheet_names = xl_workbook.sheet_names() + #print('Sheet Names', sheet_names) + # + #xl_sheet = xl_workbook.sheet_by_name(sheet_names[0]) + + # Or grab the first sheet by index + # (sheets are zero-indexed) + # First sheet is the in-game quotes + # + # xl_sheet = xl_workbook.sheet_by_index(0) + # + # + mergedListOfSubtitleSheetsAndTranslatedTREs = supportedDialogueSheets + [ x[0] for x in supportedTranslationSheets ] + + for sheetDialogueName in mergedListOfSubtitleSheetsAndTranslatedTREs: + xl_sheet = xl_workbook.sheet_by_name(sheetDialogueName) + if(xl_sheet is not None): + print ('Sheet name: %s' % xl_sheet.name) + numOfSpokenQuotes = xl_sheet.nrows - 2 # all rows minus the first TWO rows with headers + print ('num of spoken quotes: %d' % numOfSpokenQuotes) + # stats for debug + extremeQuotesList = [] + longestLength = 0 + predefinedLengthThreshold = 145 + quoteNumAboveThreshold = 0 + # end of stats for debug + + + absStartOfIndexTable = 4 + absStartOfOffsetTable = absStartOfIndexTable + (numOfSpokenQuotes * 4) # = 4 + 0x1577 * 4 = 4 + 0x55DC = 0x55E0 + absStartOfStringTable = absStartOfOffsetTable + ((numOfSpokenQuotes+1) * 4) # = 0x55E0 + (0x1578 * 4) = 0xABC0 + curStrStartOffset = absStartOfStringTable - 4 + newQuoteReplaceSpecialsAscii = '' + tmpQuoteID = 0 + #switchFlagShowQuote = False # for debugging + tmpStartFrame = 0 # for VQA sheets + tmpEndFrame = 0 # for VQA sheets + mode = 0 # init to unknown + if xl_sheet.name == supportedDialogueSheets[0]: + print 'IN GAME QUOTES' + mode = 1 #in-game quote + elif xl_sheet.name in supportedDialogueSheets: + print 'VQA SCENE DIALOGUE' + mode = 2 #VQA + elif xl_sheet.name in [ x[0] for x in supportedTranslationSheets ]: + print 'TRANSLATED TRE' + mode = 3 # Translated TRE + # + del tableOfStringIds[:] + del tableOfStringEntries[:] + del tableOfStringOffsets[:] + for row_idx in range(2, xl_sheet.nrows): + #print "Line %d" % (row_idx) + for col_idx in range(0, xl_sheet.ncols): + cell_obj = xl_sheet.cell(row_idx, col_idx) + # + # FOR IN-GAME QUOTES -- Iterate through columns starting from col 0. We need cols: 0, 2 + # + if mode == 1: + #print ('Column: [%s] cell_obj: [%s]' % (col_idx, cell_obj)) + if(col_idx == 0): + #switchFlagShowQuote = False + twoTokensfirstColSplitAtDotXLS = cell_obj.value.split('.', 1) + if len(twoTokensfirstColSplitAtDotXLS) == 2: + twoTokensfirstColSplitAtDashXLS = twoTokensfirstColSplitAtDotXLS[0].split('-', 1) + if len(twoTokensfirstColSplitAtDashXLS) == 2: + tmpQuoteID = int( twoTokensfirstColSplitAtDashXLS[0]) * 10000 + int(twoTokensfirstColSplitAtDashXLS[1]) + #print ('row_idx %d. tag %s = quoteId [%d]' % (row_idx, twoTokensfirstColSplitAtDotXLS[0], tmpQuoteID)) + tableOfStringIds.append(tmpQuoteID) + #if(tmpQuoteID == 160110 or tmpQuoteID == 160010): + # switchFlagShowQuote = True + + elif(col_idx == 1) : + #if switchFlagShowQuote == True: + # print ('length: %d: %s' % (len(cell_obj.value), cell_obj.value)) + # print ('object: %s' % (cell_obj)) + # #newQuoteReplaceSpecials = cell_obj.value.decode("utf-8") # unicode(cell_obj.value, 'windows-1252') + # #print ('decoded to unicode: %s ' % (newQuoteReplaceSpecials)) # error with char xf1 + newQuoteReplaceSpecialsAscii = translateQuoteToAsciiProper(cell_obj, xl_sheet.name) + #if switchFlagShowQuote == True: + # print ('length: %d: %s' % (len(newQuoteReplaceSpecialsAscii), newQuoteReplaceSpecialsAscii)) + #print ':'.join(x.encode('hex') for x in newQuoteReplaceSpecialsAscii) # seems to work. new chars are non-printable but exist in string + + tableOfStringEntries.append(newQuoteReplaceSpecialsAscii) + tableOfStringOffsets.append(curStrStartOffset) + curStrStartOffset += (len(newQuoteReplaceSpecialsAscii) + 1) + if ( longestLength < len(newQuoteReplaceSpecialsAscii)): + longestLength = len(newQuoteReplaceSpecialsAscii) + if ( predefinedLengthThreshold < len(newQuoteReplaceSpecialsAscii)): + extremeQuotesList.append((tmpQuoteID, newQuoteReplaceSpecialsAscii)) + quoteNumAboveThreshold += 1 + #print ('row_idx %d. tag %s = quoteId [%d], length: %d: %s' % (row_idx, twoTokensfirstColSplitAtDotXLS[0], tmpQuoteID, len(newQuoteReplaceSpecialsAscii), newQuoteReplaceSpecialsAscii)) + # + # FOR VQAs -- Iterate through columns starting from col 2. We need cols: 2, 9, 10 + # + elif mode == 2: + if(col_idx == 2): # subtitle text + newQuoteReplaceSpecialsAscii = translateQuoteToAsciiProper(cell_obj, xl_sheet.name) + #print ('length: %d: %s' % (len(newQuoteReplaceSpecialsAscii), newQuoteReplaceSpecialsAscii)) + #print ':'.join(x.encode('hex') for x in newQuoteReplaceSpecialsAscii) # seems to work. new chars are non-printable but exist in string + # don't append to tableOfStringEntries yet + elif(col_idx == 9): # startFrame + #print "cell: %s" % (cell_obj.value) + tmpStartFrame = int(cell_obj.value) + elif(col_idx == 10): # endFrame + tmpEndFrame = int(cell_obj.value) + tmpQuoteID = tmpStartFrame | (tmpEndFrame << 16) # top 16 bits are end frame (up to 65536 frames which is enough) and low 16 bits are startFrame + + tableOfStringIds.append(tmpQuoteID) + tableOfStringEntries.append(newQuoteReplaceSpecialsAscii) + tableOfStringOffsets.append(curStrStartOffset) + curStrStartOffset += (len(newQuoteReplaceSpecialsAscii) + 1) + if ( longestLength < len(newQuoteReplaceSpecialsAscii)): + longestLength = len(newQuoteReplaceSpecialsAscii) + if ( predefinedLengthThreshold < len(newQuoteReplaceSpecialsAscii)): + extremeQuotesList.append((tmpQuoteID, newQuoteReplaceSpecialsAscii)) + quoteNumAboveThreshold += 1 + # + # For translated TRE sheets the id is already in first column, the text is in the next one + # + elif mode == 3: + #print ('Column: [%s] cell_obj: [%s]' % (col_idx, cell_obj)) + if(col_idx == 0): + tmpQuoteID = int(cell_obj.value) + tableOfStringIds.append(tmpQuoteID) + elif(col_idx == 1) : + #if switchFlagShowQuote == True: + # print ('length: %d: %s' % (len(cell_obj.value), cell_obj.value)) + # print ('object: %s' % (cell_obj)) + # #newQuoteReplaceSpecials = cell_obj.value.decode("utf-8") # unicode(cell_obj.value, 'windows-1252') + # #print ('decoded to unicode: %s ' % (newQuoteReplaceSpecials)) # error with char xf1 + newQuoteReplaceSpecialsAscii = translateQuoteToAsciiProper(cell_obj, xl_sheet.name) + #if switchFlagShowQuote == True: + # print ('length: %d: %s' % (len(newQuoteReplaceSpecialsAscii), newQuoteReplaceSpecialsAscii)) + #print ':'.join(x.encode('hex') for x in newQuoteReplaceSpecialsAscii) # seems to work. new chars are non-printable but exist in string + + tableOfStringEntries.append(newQuoteReplaceSpecialsAscii) + tableOfStringOffsets.append(curStrStartOffset) + curStrStartOffset += (len(newQuoteReplaceSpecialsAscii) + 1) + if ( longestLength < len(newQuoteReplaceSpecialsAscii)): + longestLength = len(newQuoteReplaceSpecialsAscii) + if ( predefinedLengthThreshold < len(newQuoteReplaceSpecialsAscii)): + extremeQuotesList.append((tmpQuoteID, newQuoteReplaceSpecialsAscii)) + quoteNumAboveThreshold += 1 + #print ('row_idx %d. tag %s = quoteId [%d], length: %d: %s' % (row_idx, twoTokensfirstColSplitAtDotXLS[0], tmpQuoteID, len(newQuoteReplaceSpecialsAscii), newQuoteReplaceSpecialsAscii)) + + tableOfStringOffsets.append(curStrStartOffset) # the final extra offset entry + print 'Longest Length = %d, quotes above threshold (%d): %d' % (longestLength, predefinedLengthThreshold, quoteNumAboveThreshold) + for extremQuotTuple in extremeQuotesList: + print "Id: %d, Q: %s" % (extremQuotTuple[0], extremQuotTuple[1]) + # + # WRITE TO TRE FILE + # + errorFound = False + outTREFile = None + outTREFileName = sheetDialogueName[:-4] + try: + outTREFile = open("./" + outTREFileName + ".TRE", 'wb') + except: + errorFound = True + if not errorFound: + numOfSpokenQuotesToWrite = pack('I',numOfSpokenQuotes) # unsigned integer 4 bytes + outTREFile.write(numOfSpokenQuotesToWrite) + # write string IDs table + for idxe in range(0,len(tableOfStringIds)): + idOfStringToWrite = pack('I',tableOfStringIds[idxe]) # unsigned integer 4 bytes + outTREFile.write(idOfStringToWrite) + # write string offsets table + for idxe in range(0,len(tableOfStringOffsets)): + offsetOfStringToWrite = pack('I',tableOfStringOffsets[idxe]) # unsigned integer 4 bytes + outTREFile.write(offsetOfStringToWrite) + #write strings with null terminator + for idxe in range(0,len(tableOfStringEntries)): + outTREFile.write(tableOfStringEntries[idxe]) + outTREFile.write('\0') + outTREFile.close() + return +# +# +# +# ######################## +# main +# 00_0000 -- DealsInInsects dupl TLK01, TLK0A +# 00_0510 -- ThinkingOfChangingJobs-Leon dupl TLK02, TLK03 +# 00-8520 -- WhatDoYouKnow dupl TLK01, TLK0A + +# Total unique quotes seems to be 5495! +# TODO rename files in folders to conform to the underscore '_' and '-' format (a few don't -- let's have them all conforming!) +# ######################### +# +if __name__ == "__main__": + pathToQuoteExcelFile = "" + invalidSyntax = False + +# print "Len of sysargv = %s" % (len(sys.argv)) + if len(sys.argv) == 2: + if(sys.argv[1] == '--help'or sys.argv[1] == '-h'): + print "%s %s supports Blade Runner (English version, CD edition)." % (app_name_spaced, app_version) + print "Created by Praetorian of the classic adventures in Greek team." + print "Always keep backups!" + print "--------------------" + print "Preparatory steps:" + print "0. Keep actornames.txt in the same folder as this app." + print "1. Copy the BladeRunnerPCTLK.xlsx file (latest version, downloaded from Google Sheets) in some folder on your PC." + print "--------------------" + print "%s takes 1 mandatory argument:" % (app_name_spaced) + print "Valid syntax (in-game and VQA quotes): %s -x [folderpath_to_ BladeRunnerPCTLK_xlsx_file]" % (app_name) + print "1st argument is the path to the excel file with the subtitle quotes." + print "If the app finishes successfully a " + supportedDialogueSheets[0] + " and a few other .TRE files for the VQAs " + print "in the Excel file as well as a SUBTITLES.MIX file containing all of them will be created in the same folder with the app." + print "--------------------" + print "Thank you for using this app." + print "Please provide any feedback to: %s " % (company_email) + sys.exit() + elif(sys.argv[1] == '--version' or sys.argv[1] == '-v'): + print "%s %s supports Blade Runner (English version, CD edition)." % (app_name_spaced, app_version) + print "Please provide any feedback to: %s " % (company_email) + sys.exit() + else: + invalidSyntax = True + elif len(sys.argv) == 3: + if(sys.argv[1] == '-x'): + pathToQuoteExcelFile = sys.argv[2] + else: + invalidSyntax = True + + if not pathToQuoteExcelFile: + invalidSyntax = True + + if not invalidSyntax: + # parse any overrideEncoding file if exists: + initOverrideEncoding() + + + # parse the EXCEL File + # parse Actors files: + initActorPropertyEntries() +# for actorEntryTmp in actorPropertyEntries: +# print "Found actor: %s %s %s" % (actorEntryTmp[0], actorEntryTmp[1], actorEntryTmp[2]) + inputXLS(pathToQuoteExcelFile) + outputMIX() + + else: + invalidSyntax = True + + if invalidSyntax == True: + print "Invalid syntax\n Try: \n %s --help for more info \n %s --version for version info " % (app_name, app_name) + print "Valid syntax (in-game and VQA quotes): %s -x [folderpath_to_ BladeRunnerPCTLK_xlsx_file]" % (app_name) + print "1st argument is the path to the excel file with the subtitle quotes." + print "If the app finishes successfully a " + supportedDialogueSheets[0] + " and a few other .TRE files for the VQAs " + print "in the Excel file as well as a SUBTITLES.MIX file containing all of them will be created in the same folder with the app." + tmpi = 0 + for tmpArg in sys.argv: + if tmpi==0: #skip first argument + tmpi+=1 + continue + print "\nArgument: %s" % (tmpArg) + tmpi+=1 +else: + ## debug + #print '%s was imported from another module' % (app_name_spaced,) + pass diff --git a/devtools/blade_runner/subtitles/mixResourceCreator/samples/actornames.txt b/devtools/blade_runner/subtitles/mixResourceCreator/samples/actornames.txt new file mode 100644 index 0000000000..2836d18de8 --- /dev/null +++ b/devtools/blade_runner/subtitles/mixResourceCreator/samples/actornames.txt @@ -0,0 +1,75 @@ +Id Short ActorDesc #skip first row +0 MCCOY McCoy +1 STEEL Steele +2 GORDO Gordo +3 DEKTO Dektora +4 GUZZA Guzza +5 CLOVI Clovis +6 LLUCY Lucy +7 IIIZO Izo +8 SADIK Sadik +9 CRAZY Crazylegs +10 LUTHE Luther +11 GRIGO Grigorian +12 TRANS Transient +13 LANCE Lance +14 BBBOB Bullet Bob +15 RUNCI Runciter +16 INSEC Insect Dealer +17 TGUAR Tyrell Guard +18 EARLQ Early Q +19 ZUBEN Zuben +20 HASAN Hasan +21 MARCU Marcus +22 MMMIA Mia +23 OLEAR Officer Leary +24 OGRAY Officer Grayford +25 HANOI Hanoi +26 BAKER Baker +27 DCLER Desk Clerk +28 HOWIE Howie Lee +29 FISHD Fish Dealer +30 KLEIN Klein +31 MURRA Murray +32 HBARK Hawker's Barkeep +33 HOLLO Holloway +34 SWALL Sergeant Walls +35 MORAJ Moraji +36 TBARD The Bard +37 PHOTG Photographer +38 DISPA Dispatcher +39 ANSWM Answering Machine +40 RAJIF Rajif +41 GKOLV Governor Kolvig +42 ERLQB Early Q Bartender +43 HPARR Hawker's Parrot +44 TAFPA Taffy Patron +45 LOCGU Lockup Guard +46 TEENA Teenager +47 HPATA Hysteria Patron A +48 HPATB Hysteria Patron B +49 HPATC Hysteria Patron C +50 SHOES Shoeshine Man +51 TYREL Tyrell +52 CCHEW Chew +53 GGAFF Gaff +54 BRYAN Bryant +55 TAFFY Taffy +56 SEBAS Sebastian +57 RACHA Rachael +58 GDOLL General Doll +59 ISABE Isabella +60 BLIMP Blimp Guy +61 NEWSC Newscaster +62 LLEON Leon +63 MALAN Male Announcer +64 FREEA Free Slot A +65 FREEB Free Slot B +66 MAGGI Maggie +67 ACTGA Actor Genwalker A +68 ACTGB Actor Genwalker B +69 ACTGC Actor Genwalker C +70 MUTAA Mutant A +71 MUTAB Mutant B +72 MUTAC Mutant C +99 MAINF Mainframe diff --git a/devtools/blade_runner/subtitles/mixResourceCreator/samples/configureFontsTranslation.txt b/devtools/blade_runner/subtitles/mixResourceCreator/samples/configureFontsTranslation.txt new file mode 100644 index 0000000000..8b4a355ecb --- /dev/null +++ b/devtools/blade_runner/subtitles/mixResourceCreator/samples/configureFontsTranslation.txt @@ -0,0 +1,4 @@ +targetEncoding=windows-1253 +fontNameAndOutOfOrderGlyphs=SUBTLS_E#í:Ά,ñ:¥,â:¦,é:§,Ά:£ +fontNameAndOutOfOrderGlyphs=KIA6PT#é:ƒ,ü:– +fontNameAndOutOfOrderGlyphs=TAHOMA#é:ƒ,ü:–
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/mixResourceCreator/samples/sampleCMDParameters.txt b/devtools/blade_runner/subtitles/mixResourceCreator/samples/sampleCMDParameters.txt new file mode 100644 index 0000000000..71dd3c6ef8 --- /dev/null +++ b/devtools/blade_runner/subtitles/mixResourceCreator/samples/sampleCMDParameters.txt @@ -0,0 +1 @@ +python2.7 packBladeRunnerMIXFromPCTLKXLS-04.py -x ./outSpeech-02-09-2018-1358-TranslatingComms-117PatrasLinks.xls
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/module.mk b/devtools/blade_runner/subtitles/module.mk new file mode 100644 index 0000000000..7e2df5f0f3 --- /dev/null +++ b/devtools/blade_runner/subtitles/module.mk @@ -0,0 +1,2 @@ + +MODULE := devtools/blade_runner/subtitles
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/audFileDecode.py b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/audFileDecode.py new file mode 100644 index 0000000000..a46b3d8798 --- /dev/null +++ b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/audFileDecode.py @@ -0,0 +1,210 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +# +import ctypes +from struct import * + +my_module_version = "0.50" +my_module_name = "audFileDecode" + +aud_ima_index_adjust_table = [-1, -1, -1, -1, 2, 4, 6, 8] + +# aud_ima_step_table has 89 entries +aud_ima_step_table = [ + 7, 8, 9, 10, 11, 12, 13, 14, 16, + 17, 19, 21, 23, 25, 28, 31, 34, 37, + 41, 45, 50, 55, 60, 66, 73, 80, 88, + 97, 107, 118, 130, 143, 157, 173, 190, 209, + 230, 253, 279, 307, 337, 371, 408, 449, 494, + 544, 598, 658, 724, 796, 876, 963, 1060, 1166, + 1282, 1411, 1552, 1707, 1878, 2066, 2272, 2499, 2749, + 3024, 3327, 3660, 4026, 4428, 4871, 5358, 5894, 6484, + 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899, 15289, + 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767 ] + +aud_ws_step_table2 = [-2, -1, 0, 1] + +aud_ws_step_table4 = [ + -9, -8, -6, -5, -4, -3, -2, -1, + 0, 1, 2, 3, 4, 5, 6, 8 +] + +# (const xccTHA::byte* audio_in, short* audio_out, int& index, int& sample, int cs_chunk) +# index and sample are passed by reference and changed here... +# audio_out is definitely affected! +def aud_decode_ima_chunk(audioBufferIn, index, sample, cs_chunk): + code = -1 + delta = -1 + step = -1 + + audioBufferOut = [] + #for i in range(0, len(audioBufferIn)): + # print '%d: %d'%(i, int(audioBufferIn[i])) + + for sample_index in range (0, cs_chunk): + try: + code = audioBufferIn[sample_index >> 1] + except: + code = 0xa9 # dummy workaround because the c code is accessing an out of bounds index sometimes due to this shift here + #print "cs_chunk %d, sample_index %d, shifted %d, code: %d" % (cs_chunk, sample_index, sample_index >> 1, int(audioBufferIn[sample_index >> 1])) + #print "cs_chunk %s, sample_index %s, shifted %s, code: %s" % \ + # (''.join('{:04X}'.format(cs_chunk)), ''.join('{:02X}'.format(sample_index)), ''.join('{:02X}'.format(sample_index >> 1)), ''.join('{:04X}'.format(int(code)))) + code = code >> 4 if (sample_index & 1) else code & 0xf + step = aud_ima_step_table[index] + delta = step >> 3 + if (code & 1): + delta += step >> 2 + if (code & 2): + delta += step >> 1 + if (code & 4): + delta += step + if (code & 8): + sample -= delta + if (sample < -32768): + sample = -32768 + else: + sample += delta + if (sample > 32767): + sample = 32767 + audioBufferOut.append(ctypes.c_short( sample ).value ) + #audioBufferOut.append(sample) # it's not different from above... ctypes.c_short( sample ).value + #print "audio_out[%s]: %s" % (''.join('{:02X}'.format(sample_index)), ''.join('{:02X}'.format(audioBufferOut[sample_index]))); + index += aud_ima_index_adjust_table[code & 7] + if (index < 0): + index = 0 + elif (index > 88): + index = 88 + ## output buffer of shorts + #binDataOut = struct.pack('h'*len(audioBufferOut), *audioBufferOut) + #return (binDataOut, index, sample) + return (audioBufferOut, index, sample) +# +# +# +def aud_decode_clip8(v): + if (v < 0): + return 0 + return 0xff if (v > 0xff) else v +# +# +# + +# (const xccTHA::byte* r, char* w, int cb_s, int cb_d) +def aud_decode_ws_chunk(inputChunkBuffer, cb_s, cb_d): + outputChunkBuffer = [] + inpChBuffIter = 0 + outChBuffIter = 0 + + if (cb_s == cb_d): + # outputChunkBuffer = inputChunkBuffer[:cb_s] # memcpy(w, r, cb_s) # FIX + for mcp in range(0, cb_s): + outputChunkBuffer.append(ctypes.c_char(inputChunkBuffer[inpChBuffIter + mcp]).value) + #binDataOut = struct.pack('b'*len(outputChunkBuffer), *outputChunkBuffer) + #return binDataOut + return outputChunkBuffer + +# const xccTHA::byte* s_end = inputChunkBuffer + cb_s; # FIX + + s_end = inpChBuffIter + cb_s + sample = ctypes.c_int(0x80).value #int sample + while (inpChBuffIter < s_end): + inpChBuffIter += 1 + count = ctypes.c_char(inputChunkBuffer[inpChBuffIter] & 0x3f).value # char count + switchKey = inputChunkBuffer[inpChBuffIter - 1] >> 6 # inputChunkBuffer[-1] # b[-1] is *(b - 1) + if switchKey == 0: + count += 1 + for iter in range (count, 0, -1): + inpChBuffIter += 1 + code = ctypes.c_int(inputChunkBuffer[inpChBuffIter]).value # int code + # assignment in C was right to left so: + # *(outputChunkBuffer++) = sample = clip8(sample + aud_ws_step_table2[code & 3]) + # is: + # *(outputChunkBuffer++) = (sample = clip8(sample + aud_ws_step_table2[code & 3])) + # which is equivalent to these two commands: + # sample = clip8(sample + aud_ws_step_table2[code & 3]) + # *(outputChunkBuffer++) = sample + # SO: + sample = aud_decode_clip8(sample + aud_ws_step_table2[code & 3]) + outputChunkBuffer.append(ctypes.c_char(sample).value) + outChBuffIter += 1 + sample = aud_decode_clip8(sample + aud_ws_step_table2[code >> 2 & 3]) + outputChunkBuffer.append(ctypes.c_char(sample).value) + outChBuffIter += 1 + sample = aud_decode_clip8(sample + aud_ws_step_table2[code >> 4 & 3]) + outputChunkBuffer.append(ctypes.c_char(sample).value) + outChBuffIter += 1 + sample = aud_decode_clip8(sample + aud_ws_step_table2[code >> 6]) + outputChunkBuffer.append(ctypes.c_char(sample).value) + outChBuffIter += 1 + elif switchKey == 1: + count += 1 + for iter in range (count, 0, -1): + inpChBuffIter += 1 + code = inputChunkBuffer[inpChBuffIter] # int code + sample += aud_ws_step_table4[code & 0xf] + sample = aud_decode_clip8(sample) + outputChunkBuffer.append(ctypes.c_char(sample).value) + outChBuffIter += 1 + sample += aud_ws_step_table4[code >> 4] + sample = aud_decode_clip8(sample) + outputChunkBuffer.append(ctypes.c_char(sample).value) + outChBuffIter += 1 + elif switchKey == 2: + if (count & 0x20): + #sample += static_cast<char>(count << 3) >> 3 + #*(outputChunkBuffer++) = sample + sample += ((count & 0xFF) << 3 ) >> 3 + outputChunkBuffer.append(ctypes.c_char(sample).value) + outChBuffIter += 1 + else: + count += 1 + # memcpy(outputChunkBuffer, inputChunkBuffer, count) # FIX + for mcp in range(0, count): + outputChunkBuffer.append(ctypes.c_char(inputChunkBuffer[inpChBuffIter + mcp]).value) + inpChBuffIter += count + outChBuffIter += count + sample = inputChunkBuffer[inpChBuffIter - 1] + else: + count += 1 + # memset(outputChunkBuffer, sample, ++count) + for mst in range(0, count): + outputChunkBuffer.append(ctypes.c_char(sample).value) + outChBuffIter += count; + # output buffer of chars + #binDataOut = struct.pack('b'*len(outputChunkBuffer), *outputChunkBuffer) + #return binDataOut + return outputChunkBuffer + +# +# +# + +class audFileDecode: + m_index = -1 + m_sample = -1 + + def __init__(self, index = 0, sample = 0): + self.m_index = index; + self.m_sample = sample; + return + + def index(self): + return self.m_index + + # (const xccTHA::byte* audio_in, short* audio_out, int cs_chunk) + def decode_chunk(self, audio_in, cs_chunk): + (audio_Out, outIndex, outSample) = aud_decode_ima_chunk(audio_in, self.m_index, self.m_sample, cs_chunk) + self.m_index = outIndex + self.m_sample = outSample + return audio_Out + +if __name__ == '__main__': + # main() + print "Running %s as main module" % (my_module_name) + decodeInstance = audFileDecode() + +else: + #debug + #print "Running %s imported from another module" % (my_module_name) + pass +
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/audFileLib.py b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/audFileLib.py new file mode 100644 index 0000000000..1adf4bc969 --- /dev/null +++ b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/audFileLib.py @@ -0,0 +1,289 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +# +import os, sys, shutil +import wave +import struct +from struct import * +from audFileDecode import * +import ctypes + +my_module_version = "0.50" +my_module_name = "audFileLib" + +#constants +aud_chunk_id = 0x0000deaf +SIZE_OF_AUD_HEADER_IN_BYTES = 12 +SIZE_OF_AUD_CHUNK_HEADER_IN_BYTES = 8 + +class AudHeader: + samplerate = -1 #// Frequency // int16_t // TODO should be unsigned (?) + size_in = -1 #// Size of file (without header) // int32_t // TODO should be unsigned (?) + size_out = -1 #// Size of output data // int32_t // TODO should be unsigned (?) + flags = -1 #// bit 0=stereo, bit 1=16bit // int8_t + compression = -1 #// 1=WW compressed, 99=IMA ADPCM (0x63) // int8_t + + def __init__(self): + return + + +#//The rest of the AUD files is divided in chunks. These are usually 512 +#//bytes long, except for the last one. +class AudChunkHeader: + size_in = -1 #// Size of compressed data // int16_t // TODO should be unsigned (?) + size_out = -1 #// Size of output data // int16_t // TODO should be unsigned (?) + id = 0x0000FFFF #// Always 0x0000DEAF // int32_t + + def __init__(self): + return + +# +# +# +class audFile: + m_header = AudHeader() + def __init__(self): + return + + # std::fstream& fs, AudFileNS::pos_type startAudFilepos, AudFileNS::pos_type endAudFilepos, const std::string& filename + def extract_as_wav(self, audBytesBuff, filename): + print "Saving to wav: " + filename + + cvirtualBinaryD = self.decode(audBytesBuff) +# TODO DEBUG REMOVED FOR NOW. TODO RESTORE THIS!!! +# if (not cvirtualBinaryD): +# return 1 + + cb_sample = self.get_cb_sample() + cs_remaining = self.get_c_samples() + + waveWritFile = wave.open(filename, 'wb') + waveWritFile.setnchannels(self.get_c_channels()) + waveWritFile.setsampwidth(cb_sample) + waveWritFile.setframerate(self.get_samplerate()) + waveWritFile.setnframes(cs_remaining) + #waveWritFile.setcomptype(None, '') + waveWritFile.writeframesraw(cvirtualBinaryD) + waveWritFile.close() + +# t_wav_header header; +# memset(&header, 0, sizeof(t_wav_header)); +# header.file_header.id = wav_file_id; // # "RIFF" +# header.file_header.size = sizeof(header) - sizeof(header.file_header) + (cs_remaining << 1); +# header.form_type = wav_form_id; // # "WAVE" +# header.format_chunk.header.id = wav_format_id; // #"fmt " +# header.format_chunk.header.size = sizeof(header.format_chunk) - sizeof(header.format_chunk.header); +# header.format_chunk.formattag = 1; +# header.format_chunk.c_channels = 1; +# header.format_chunk.samplerate = get_samplerate(); +# header.format_chunk.byterate = cb_sample * get_samplerate(); +# header.format_chunk.blockalign = cb_sample; +# header.format_chunk.cbits_sample = cb_sample << 3; +# header.data_chunk_header.id = wav_data_id; # "data" +# header.data_chunk_header.size = cb_sample * cs_remaining; +# error = f.write(&header, sizeof(t_wav_header)); +# return error ? error : f.write(d); + return 0 # TODO fix + + + def loadAudFile(self, audBytesBuff, maxLength): + offsInAudFile = 0 + tmpTuple = struct.unpack_from('h', audBytesBuff, offsInAudFile) + self.header().samplerate = tmpTuple[0] + offsInAudFile += 2 + tmpTuple = struct.unpack_from('i', audBytesBuff, offsInAudFile) + self.header().size_in = tmpTuple[0] + offsInAudFile += 4 + tmpTuple = struct.unpack_from('i', audBytesBuff, offsInAudFile) + self.header().size_out = tmpTuple[0] + offsInAudFile += 4 + tmpTuple = struct.unpack_from('b', audBytesBuff, offsInAudFile) + self.header().flags = tmpTuple[0] + offsInAudFile += 1 + tmpTuple = struct.unpack_from('b', audBytesBuff, offsInAudFile) + self.header().compression = tmpTuple[0] + offsInAudFile += 1 + print "samplerate: %d\tsizeIn: %d\tsizeOut: %d\tflags: %d\tcompression: %d" % (self.get_samplerate(), self.header().size_in, self.header().size_out, self.header().flags, self.header().compression) + + if self.get_samplerate() < 8000 or self.get_samplerate() > 48000 or self.header().size_in > (maxLength - SIZE_OF_AUD_HEADER_IN_BYTES ): + print "AUD HEADER SIZE ERROR::2" + return False + else: + if self.header().compression == 1: + if (self.header().flags != 0): + return False + elif self.header().compression == 0x63: + if (self.header().flags != 2): + return False + return True + + # int AudFile::get_chunk_header(int i, std::fstream& fs, AudFileNS::pos_type startAudFilepos, AudFileNS::pos_type endAudFilepos, AudChunkHeader& outAudChunkHeader) + def get_chunk_header(self, chunkIdx, inAudFileBytesBuffer, inAudFileSize ): + #fs.seekg(int(startAudFilepos) + int(SIZE_OF_AUD_HEADER_IN_BYTES), fs.beg); + #AudFileNS::pos_type rAudPos; + #rAudPos = fs.tellg(); + outAudChunkHeader = AudChunkHeader() + rAudPos = SIZE_OF_AUD_HEADER_IN_BYTES + + #AudChunkHeader tmpInremediateChunkheader; + tmpInremediateChunkheader = AudChunkHeader() + #while (i--) # value of i is decreased after checked by while loop + while(chunkIdx > 0): + chunkIdx -= 1 + if (rAudPos + SIZE_OF_AUD_CHUNK_HEADER_IN_BYTES > inAudFileSize): + return (-1, rAudPos, None) + + tmpAudFileOffset = rAudPos + tmpTuple = struct.unpack_from('h', inAudFileBytesBuffer, tmpAudFileOffset) + tmpInremediateChunkheader.size_in = tmpTuple[0] + tmpAudFileOffset += 2 + tmpTuple = struct.unpack_from('h', inAudFileBytesBuffer, tmpAudFileOffset) + tmpInremediateChunkheader.size_out = tmpTuple[0] + tmpAudFileOffset += 2 + tmpTuple = struct.unpack_from('i', inAudFileBytesBuffer, tmpAudFileOffset) + tmpInremediateChunkheader.id = tmpTuple[0] + tmpAudFileOffset += 4 + #fs.read((char*)&tmpInremediateChunkheader, SIZE_OF_AUD_CHUNK_HEADER_IN_BYTES); + rAudPos += SIZE_OF_AUD_CHUNK_HEADER_IN_BYTES + tmpInremediateChunkheader.size_in + #fs.seekg(int(rAudPos), fs.beg); + + if (rAudPos + SIZE_OF_AUD_CHUNK_HEADER_IN_BYTES > inAudFileSize ): + return (-1, rAudPos, None) + # write to FINAL output chunk header + tmpAudFileOffset = rAudPos + tmpTuple = struct.unpack_from('h', inAudFileBytesBuffer, tmpAudFileOffset) + outAudChunkHeader.size_in = tmpTuple[0] + tmpAudFileOffset += 2 + tmpTuple = struct.unpack_from('h', inAudFileBytesBuffer, tmpAudFileOffset) + outAudChunkHeader.size_out = tmpTuple[0] + tmpAudFileOffset += 2 + tmpTuple = struct.unpack_from('i', inAudFileBytesBuffer, tmpAudFileOffset) + outAudChunkHeader.id = tmpTuple[0] + tmpAudFileOffset += 4 + #fs.read((char*)&outAudChunkHeader, SIZE_OF_AUD_CHUNK_HEADER_IN_BYTES); + if (rAudPos + SIZE_OF_AUD_CHUNK_HEADER_IN_BYTES + outAudChunkHeader.size_in > inAudFileSize): + return (-1, rAudPos, None) + rAudPos += SIZE_OF_AUD_CHUNK_HEADER_IN_BYTES + return (0, rAudPos, outAudChunkHeader) # //reinterpret_cast<const AudChunkHeader*>(r); + + # int AudFile::get_chunk_data(int i, std::fstream& fs, int sizeToRead, AudFileNS::byte* byteChunkDataPtr) + def get_chunk_data(self, inAudFileBytesBuffer, startOffs, sizeToRead): + #fs.read((char*)byteChunkDataPtr, sizeToRead) + outChunkDataLst = [] + #print "startOffs %d, sizeToRead %d" % (startOffs, sizeToRead) + for i in range(startOffs, startOffs + sizeToRead): + #outChunkDataLst.append(ctypes.c_char(inAudFileBytesBuffer[i]).value) + #outChunkDataLst.append(ctypes.c_byte(inAudFileBytesBuffer[i]).value) + tmpTuple = struct.unpack_from('b', inAudFileBytesBuffer, i) + outChunkDataLst.append(tmpTuple[0]) + #byteChunkDataOut = struct.pack('b'*len(outChunkDataLst), *outChunkDataLst) + #return (0, byteChunkDataOut) + return (0, outChunkDataLst) + + + # std::fstream& fs, AudFileNS::pos_type startAudFilepos, AudFileNS::pos_type endAudFilepos + # returned Cvirtual_binary + def decode(self, audBytesBuff): + # The * operator unpacks an argument list. It allows you to call a function with the list items as individual arguments. + # binDataOut = struct.pack('i'*len(data), *data) + print "DECODING..." +# Cvirtual_binary d; + binaryDataOutLst = [] + binaryDataOutBuff = None + cb_audio = self.get_cb_sample() * self.get_c_samples() # int cb_audio - basically this should be the size_out + if self.header().compression == 1: + # write_start allocates space for virtualBinary + # AudFileNS::byte* w = d.write_start(cb_audio); + errGetChunk = 0 # int errGetChunk + #for (int chunk_i = 0; w != d.data_end(); chunk_i++) + chunk_i = 0 + wIndex = 0 + while (wIndex < cb_audio): + #AudChunkHeader out_chunk_header; + #out_chunk_header = AudChunkHeader() + (errGetChunk, bufferDataPos, out_chunk_header) = self.get_chunk_header(chunk_i, audBytesBuff, len(audBytesBuff)) + if errGetChunk != 0: +# print "Error OR End file case while getting uncompressed chunk header!" + break + #print "Get uncompressed chunk header returned:: %d " % (out_chunk_header.id) + #Cvirtual_binary out_chunk_data; + #AudFileNS::byte* byteChunkDataPtr = out_chunk_data.write_start(out_chunk_header.size_in); + (errorGCD, byteChunkDataLst) = self.get_chunk_data(audBytesBuff, bufferDataPos, out_chunk_header.size_in) + # export decoded chunk to w (output) buffer (of CHARS) at the point where we're currently at (so append there) + decodedAudioChunkAsLst = aud_decode_ws_chunk(byteChunkDataLst, out_chunk_header.size_in, out_chunk_header.size_out) + binaryDataOutLst.extend(decodedAudioChunkAsLst) + wIndex += out_chunk_header.size_out + chunk_i += 1 + binaryDataOutBuff = struct.pack('b'*len(binaryDataOutLst), *binaryDataOutLst) + elif self.header().compression == 0x63: + decodeInstance = audFileDecode(); + #decodeInstance.init(); + #AudFileNS::byte* w = d.write_start(cb_audio); + errGetChunk = 0 # int errGetChunk + # for (int chunk_i = 0; w != d.data_end(); chunk_i++) + chunk_i = 0 + wIndex = 0 + while (wIndex < cb_audio): + #print("chunkI: %d\t Windex: %d\t cb_audio: %d") % (chunk_i,wIndex,cb_audio) + #AudChunkHeader out_chunk_header; + #out_chunk_header = AudChunkHeader() + #errGetChunk = self.get_chunk_header(chunk_i, fs, startAudFilepos, endAudFilepos, out_chunk_header); + (errGetChunk, bufferDataPos, out_chunk_header) = self.get_chunk_header(chunk_i, audBytesBuff, len(audBytesBuff)) + if errGetChunk != 0: + print "Error OR End file case while getting COMPRESSED chunk header!" + break + #print "Get COMPRESSED chunk header returned:: headerInSize: %d headerOutSize: %d id: %d" % (out_chunk_header.size_in, out_chunk_header.size_out, out_chunk_header.id) + #Cvirtual_binary out_chunk_data; + #AudFileNS::byte* byteChunkDataPtr = out_chunk_data.write_start(out_chunk_header.size_in); + (errorGCD, byteChunkDataLst) = self.get_chunk_data(audBytesBuff, bufferDataPos, out_chunk_header.size_in) + # export decoded chunk to w (output) buffer (of SHORTS) at the point where we're currently at (so append there) + #print "byteChunkDataLst len: %d, size_in was: %d" % (len(byteChunkDataLst), out_chunk_header.size_in) + decodedAudioChunkAsLst = decodeInstance.decode_chunk(byteChunkDataLst, out_chunk_header.size_out / self.get_cb_sample()); + binaryDataOutLst.extend(decodedAudioChunkAsLst) + wIndex += out_chunk_header.size_out + #print("new Windex: %d\t cb_audio: %d") % (wIndex,cb_audio) + chunk_i += 1 + binaryDataOutBuff = struct.pack('h'*len(binaryDataOutLst), *binaryDataOutLst) + return binaryDataOutBuff + + def header(self): + return self.m_header + + def get_c_samples(self): + return self.m_header.size_out / self.get_cb_sample() + + def get_samplerate(self): + return self.m_header.samplerate; + + # flag bit 0 is stereo(set) mono(clear) + def get_c_channels(self): + return 2 if (self.m_header.flags & 0x01) else 1; + + # flag bit 1 is 16bit(set) 8bit (clear) + def get_cb_sample(self): + return 2 if (self.m_header.flags & 0x02) else 1 +# +# +# +if __name__ == '__main__': + # main() + print "Running %s as main module" % (my_module_name) + # assumes a file of name 000000.AUD in same directory + inAUDFile = None + errorFound = False + try: + inAUDFile = open(os.path.join('.','00000000.AUD'), 'rb') + except: + errorFound = True + print "Unexpected error:", sys.exc_info()[0] + raise + if not errorFound: + allOfAudFileInBuffer = inAUDFile.read() + audFileInstance = audFile() + audFileInstance.loadAudFile(allOfAudFileInBuffer, len(allOfAudFileInBuffer)) + audFileInstance.extract_as_wav(allOfAudFileInBuffer, './tmp.wav') + inAUDFile.close() +else: + #debug + #print "Running %s imported from another module" % (my_module_name) + pass
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/samples/actornames.txt b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/samples/actornames.txt new file mode 100644 index 0000000000..2836d18de8 --- /dev/null +++ b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/samples/actornames.txt @@ -0,0 +1,75 @@ +Id Short ActorDesc #skip first row +0 MCCOY McCoy +1 STEEL Steele +2 GORDO Gordo +3 DEKTO Dektora +4 GUZZA Guzza +5 CLOVI Clovis +6 LLUCY Lucy +7 IIIZO Izo +8 SADIK Sadik +9 CRAZY Crazylegs +10 LUTHE Luther +11 GRIGO Grigorian +12 TRANS Transient +13 LANCE Lance +14 BBBOB Bullet Bob +15 RUNCI Runciter +16 INSEC Insect Dealer +17 TGUAR Tyrell Guard +18 EARLQ Early Q +19 ZUBEN Zuben +20 HASAN Hasan +21 MARCU Marcus +22 MMMIA Mia +23 OLEAR Officer Leary +24 OGRAY Officer Grayford +25 HANOI Hanoi +26 BAKER Baker +27 DCLER Desk Clerk +28 HOWIE Howie Lee +29 FISHD Fish Dealer +30 KLEIN Klein +31 MURRA Murray +32 HBARK Hawker's Barkeep +33 HOLLO Holloway +34 SWALL Sergeant Walls +35 MORAJ Moraji +36 TBARD The Bard +37 PHOTG Photographer +38 DISPA Dispatcher +39 ANSWM Answering Machine +40 RAJIF Rajif +41 GKOLV Governor Kolvig +42 ERLQB Early Q Bartender +43 HPARR Hawker's Parrot +44 TAFPA Taffy Patron +45 LOCGU Lockup Guard +46 TEENA Teenager +47 HPATA Hysteria Patron A +48 HPATB Hysteria Patron B +49 HPATC Hysteria Patron C +50 SHOES Shoeshine Man +51 TYREL Tyrell +52 CCHEW Chew +53 GGAFF Gaff +54 BRYAN Bryant +55 TAFFY Taffy +56 SEBAS Sebastian +57 RACHA Rachael +58 GDOLL General Doll +59 ISABE Isabella +60 BLIMP Blimp Guy +61 NEWSC Newscaster +62 LLEON Leon +63 MALAN Male Announcer +64 FREEA Free Slot A +65 FREEB Free Slot B +66 MAGGI Maggie +67 ACTGA Actor Genwalker A +68 ACTGB Actor Genwalker B +69 ACTGC Actor Genwalker C +70 MUTAA Mutant A +71 MUTAB Mutant B +72 MUTAC Mutant C +99 MAINF Mainframe diff --git a/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/samples/outSpeech-01-11-2018-1414.xls b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/samples/outSpeech-01-11-2018-1414.xls Binary files differnew file mode 100644 index 0000000000..1a70a220c6 --- /dev/null +++ b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/samples/outSpeech-01-11-2018-1414.xls diff --git a/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/samples/sampleCMDParameters.txt b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/samples/sampleCMDParameters.txt new file mode 100644 index 0000000000..a41e6c291a --- /dev/null +++ b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/samples/sampleCMDParameters.txt @@ -0,0 +1 @@ +python2.7 sortBladeRunnerWavs02.py -op F:\WORKSPACE_\BladeRunnerExtrTools\br-mixer-master\data\WAV -ip H:/Games/BladeRunner -xtre
\ No newline at end of file diff --git a/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/sortBladeRunnerWavs02.py b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/sortBladeRunnerWavs02.py new file mode 100644 index 0000000000..4bc5bb7e23 --- /dev/null +++ b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/sortBladeRunnerWavs02.py @@ -0,0 +1,722 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +# +# Created by Praetorian (ShadowNate) for Classic Adventures in Greek +# classic.adventures.in.greek@gmail.com +# +# TODO update README +# TODO test recreation of TRE file (Especially in Credits which the original has a few special characters (font delegates)) +# +# DONE Add code and switch option: to get the blade runner installation directory as input, then find the TLK files and extract them with proper naming +# DONE fix proper names for sheets as per latest code changes +# +import os, sys, shutil +from os import walk, errno +import xlwt +import csv +import os.path +from xlwt import * +from audFileLib import * +from treFileLib import * + +# encoding=utf8 +#reload(sys) +#sys.setdefaultencoding('utf8') + +company_email = "classic.adventures.in.greek@gmail.com" +app_version = "0.50" +app_name = "sortBladeRunnerWavs" +app_name_spaced = "Sort Blade Runner Audio Speech Files" +stringReplacementForRootFolderWithExtractedFiles = "" +numReplaceStartingCharacters = 0 + +OUTPUT_XLS_FILENAME = 'out.xls' +OUTPUT_XLS_QUOTES_SHEET = 'INGQUO_E.TRE' + +supportedTLKInputFiles = [('1.TLK', 'TLK01'), ('2.TLK', 'TLK02'), ('3.TLK', 'TLK03'), ('A.TLK', 'TLK0A'), ('SPCHSFX.TLK', 'TLKSPCHSFX')] +supportedMIXInputFiles = ['STARTUP.MIX'] +# 15 TRE files +supportedExportedTREFiles = ['CLUES.TRE','ACTORS.TRE','CRIMES.TRE','CLUETYPE.TRE','KIA.TRE','SPINDEST.TRE','VK.TRE','OPTIONS.TRE','DLGMENU.TRE','ENDCRED.TRE','HELP.TRE','SCORERS.TRE','KIACRED.TRE','ERRORMSG.TRE','AUTOSAVE.TRE'] + +wavfiles = [] +wavfilesNoDups = [] +actorPropertyEntries = [] #[0]:id, [1]:ShortHand Name [2]:Full Name +actorPropertyEntriesWasInit = False + + +# strFileName should be the full file name (including extension) +def calculateFoldHash(strFileName): + i = 0 + hash = 0 + strParam = strFileName.upper() + lenFileName = len(strParam); + while i < lenFileName and i < 12: + groupSum = 0 + # work in groups of 4 bytes + for j in range(0, 4): + # LSB first, so the four letters in the string are re-arranged (first letter goes to lower place) + groupSum >>= 8; + if (i < lenFileName): + groupSum |= (ord(strParam[i]) << 24) + i += 1 + else: # if i >= lenFileName but still haven't completed the four byte loop add 0s + groupSum |= 0 + hash = ((hash << 1) | ((hash >> 31) & 1)) + groupSum + hash &= 0xFFFFFFFF # mask here! + #print (strParam +': ' +''.join('{:08X}'.format(hash))) + return hash + +# Fill the actorPropertyEntries table +def initActorPropertyEntries(): + global actorPropertyEntriesWasInit + global actorPropertyEntries + firstLine = True +# print "opening actornames" + with open("./actornames.txt") as tsv: + for line in csv.reader(tsv, dialect="excel-tab"): + #skip first line header + if firstLine == True: +# print "skippingHeader" + firstLine = False + else: + actorPropertyEntries.append(line) + actorPropertyEntriesWasInit = True + tsv.close() + +def getActorShortNameById(lookupActorId): + global actorPropertyEntriesWasInit + global actorPropertyEntries + if not actorPropertyEntriesWasInit: + return '' + else: + for actorEntryTmp in actorPropertyEntries: + if int(actorEntryTmp[0]) == int(lookupActorId): + return actorEntryTmp[1] + return '' + + +def getActorFullNameById(lookupActorId): + global actorPropertyEntriesWasInit + global actorPropertyEntries + if not actorPropertyEntriesWasInit: + return '' + else: + for actorEntryTmp in actorPropertyEntries: + if int(actorEntryTmp[0]) == int(lookupActorId): + return actorEntryTmp[2] + return '' + +def getActorIdByShortName(lookupActorShortName): + global actorPropertyEntriesWasInit + global actorPropertyEntries + if not actorPropertyEntriesWasInit: + return '' + else: + for actorEntryTmp in actorPropertyEntries: + if actorEntryTmp[1] == lookupActorShortName: + return actorEntryTmp[0].zfill(2) + return '' + +def getActorShortNameAndLocalQuoteIdByAUDHashID(audHashId): + actorId = 0 + actorShortName = '' + actorLocalQuoteId = 0 + if not actorPropertyEntriesWasInit: + print "Error actor properties were not initialized!" + return (actorId, actorShortName, actorLocalQuoteId) + + for actorEntryTmp in actorPropertyEntries: + if( (audHashId - (int(actorEntryTmp[0]) * 10000) ) >= 0) and ((audHashId - (int(actorEntryTmp[0]) * 10000)) < 10000): + actorId = int(actorEntryTmp[0]) + actorShortName = actorEntryTmp[1] + actorLocalQuoteId = audHashId - (actorId * 10000) + return (actorId, actorShortName, actorLocalQuoteId) + return (actorId, actorShortName, actorLocalQuoteId) + +# Aux. Ensure existence of output directory +def ensure_dir(directory): + try: + os.makedirs(directory) + except OSError as e: + if e.errno != errno.EEXIST: + raise + +# +# Reading in the INPUT TLK files and checking all the AUD file properties +# +def inputTLKsExtract(inputTLKpath, outputWAVpath): + # try to open all TLK file entries from supportedTLKInputFiles + # then per TLK file + # create an output folder in the OUTPUT PATH named TLK## for the 1, 2, 3 TLK and TLKSPCHSFX for the SPCHSFX.TLK + # printout: + # total entries + # total data size + # and per entry the + # fileID + # segment offset + # file size + print "Checking in %s for TLK files to extract to %s" % (inputTLKpath, outputWAVpath) + inputTLKFilesFound = [] + # breaking after first for loop yields only the top directory files, which is what we want + for (dirpath, dirnames, filenames) in walk(inputTLKpath): + for filename in filenames: + for tlkTuple in supportedTLKInputFiles: + if filename.upper() == tlkTuple[0]: + inputTLKFilesFound.append(tlkTuple) + break + for tmpTLKfileTuple in inputTLKFilesFound: + print "Found TLK: %s" % ('"' + inputTLKpath + tmpTLKfileTuple[0] + '"') + errorFound = False + inTLKFile = None + # + # Create output folder if not exists at output path + print "Ensuring output directory %s" % (os.path.join(outputWAVpath, tmpTLKfileTuple[1] )) + ensure_dir(os.path.join(outputWAVpath, tmpTLKfileTuple[1] ) ) + try: + inTLKFile = open(os.path.join(inputTLKpath,tmpTLKfileTuple[0]), 'rb') + except: + errorFound = True + print "Unexpected error:", sys.exc_info()[0] + raise + if not errorFound: + tmpBuff = inTLKFile.read(2) + # H: unsigned short (2 bytes) followed by I: unsigned int (4 bytes) + tlkFileEntriesNumTuple = struct.unpack('H', tmpBuff) + numOfEntriesToExtract = tlkFileEntriesNumTuple[0] + tmpBuff = inTLKFile.read(4) + tlkFileDataSegmentSizeTuple = struct.unpack('I', tmpBuff) + allTlkFileSize = tlkFileDataSegmentSizeTuple[0] + inTLKFile.seek(0, 2) # go to file end + allActualBytesInMixFile = inTLKFile.tell() + inTLKFile.seek(6, 0) # go to start of table of TLK file entries (right after the 6 bytes header) + # 2 + 4 = 6 bytes short MIX header + # 12 bytes per TLK entry in entries table + # quick size validation + print "Entries: %d, data segment %d bytes" % (numOfEntriesToExtract, allTlkFileSize) + if allActualBytesInMixFile != 2 + 4 + 12 * numOfEntriesToExtract + allTlkFileSize: + print "Error: TLK file size mismatch with reported size in header for %s!" % (tmpTLKfileTuple[0]) + else: + # + # 12 bytes per entry + # 4 bytes: ID + # 4 bytes: Offset in data segment + # 4 bytes: Size of data + # + for i in range(0, numOfEntriesToExtract): + inTLKFile.seek(2 + 4 + 12*i) + tmpBuff = inTLKFile.read(4) + tmpRdTuple = struct.unpack('I', tmpBuff) + idOfAUDEntry = tmpRdTuple[0] + tmpBuff = inTLKFile.read(4) + tmpRdTuple = struct.unpack('I', tmpBuff) + offsetOfAUDEntry = tmpRdTuple[0] + tmpBuff = inTLKFile.read(4) + tmpRdTuple = struct.unpack('I', tmpBuff) + sizeOfAUDEntry = tmpRdTuple[0] + print "Entry: %s, offset %s, data segment %s bytes" % (''.join('{:08X}'.format(idOfAUDEntry)), ''.join('{:08X}'.format(offsetOfAUDEntry)),''.join('{:08X}'.format(sizeOfAUDEntry))) + # + # put file in AUD object + # do we need AUD decode? + # create WAV from AUD + # write WAV to appropriate output folder + # Figure out proper naming for file + # then: + # AudFile aud; + # aud.loadAudFile(fs); (fs is file stream) + # aud.extract_as_wav(fs, offset, offset + int(sizeof(AudHeader)) + aud.header().size_in, target); + # + # + inTLKFile.seek(2 + 4 + 12*numOfEntriesToExtract + offsetOfAUDEntry) + if(offsetOfAUDEntry + sizeOfAUDEntry > allTlkFileSize): + print "Error: AUD file size mismatch with reported size in entry header!" + else: + audFileBuffer = inTLKFile.read(sizeOfAUDEntry) + if (len(audFileBuffer) == sizeOfAUDEntry): + # load Aud file + thisAudFile = audFile() + if (thisAudFile.loadAudFile(audFileBuffer, allTlkFileSize)): + # print "AUD file load successful!" + # find + # print "Emulating Wav write to appropriate folder..." + (actorID, actorSName, localQuoteId) = getActorShortNameAndLocalQuoteIdByAUDHashID(idOfAUDEntry) + targetSimpleFileName = actorSName + '_' + str(localQuoteId).zfill(4) + '_' + ''.join('{:08X}'.format(idOfAUDEntry)).upper()+'.WAV' + #print os.path.join(outputWAVpath, tmpTLKfileTuple[1], targetSimpleFileName) + if not os.path.isfile(os.path.join(outputWAVpath, tmpTLKfileTuple[1], targetSimpleFileName) ): + thisAudFile.extract_as_wav(audFileBuffer, os.path.join(outputWAVpath, tmpTLKfileTuple[1], targetSimpleFileName) ) + else: + print "Output file %s already exists. Skipping..." % (os.path.join(outputWAVpath, tmpTLKfileTuple[1], targetSimpleFileName)) + else: + print "Error while LOADING aud file!" + else: + print "Error while reading AUD file %s into mem buffer" % (''.join('{:08X}'.format(idOfAUDEntry))) + inTLKFile.close() + + + # SYS EXIT IS HERE ONLY FOR DEBUG PURPOSES OF PARSING TLK FILES - SHOULD BE COMMENTED OUT NORMALLY + # sys.exit(0) + return + +def inputMIXExtractTREs(inputMIXpath, excelOutBook = None): + print "Checking in %s for MIX files to extract TRE's from" % (inputMIXpath) + inputMIXFilesFound = [] + # breaking after first for loop yields only the top directory files, which is what we want + for (dirpath, dirnames, filenames) in walk(inputMIXpath): + for filename in filenames: + for mixFileName in supportedMIXInputFiles: + if filename.upper() == mixFileName: + inputMIXFilesFound.append(mixFileName) + break + for tmpMIXfileName in inputMIXFilesFound: + print "Found MIX: %s" % ('"' + inputMIXpath + tmpMIXfileName + '"') + errorFound = False + inMIXFile = None + # + try: + inMIXFile = open(os.path.join(inputMIXpath,tmpMIXfileName), 'rb') + except: + errorFound = True + print "Unexpected error:", sys.exc_info()[0] + raise + if not errorFound: + totalTREs = 0 + tmpBuff = inMIXFile.read(2) + # H: unsigned short (2 bytes) followed by I: unsigned int (4 bytes) + mixFileEntriesNumTuple = struct.unpack('H', tmpBuff) + numOfEntriesToExtract = mixFileEntriesNumTuple[0] + tmpBuff = inMIXFile.read(4) + mixFileDataSegmentSizeTuple = struct.unpack('I', tmpBuff) + allMixFileSize = mixFileDataSegmentSizeTuple[0] + inMIXFile.seek(0, 2) # go to file end + allActualBytesInMixFile = inMIXFile.tell() + inMIXFile.seek(6, 0) # go to start of table of MIX file entries (right after the 6 bytes header) + # 2 + 4 = 6 bytes short MIX header + # 12 bytes per MIX entry in entries table + # quick size validation + print "Entries: %d, data segment %d bytes" % (numOfEntriesToExtract, allMixFileSize) + if allActualBytesInMixFile != 2 + 4 + 12 * numOfEntriesToExtract + allMixFileSize: + print "Error: MIX file size mismatch with reported size in header for %s!" % (tmpMIXfileName) + else: + # + # 12 bytes per entry + # 4 bytes: ID + # 4 bytes: Offset in data segment + # 4 bytes: Size of data + # + for i in range(0, numOfEntriesToExtract): + foundTREFile = False + currTreFileName = 'UNKNOWN.TRE' + inMIXFile.seek(2 + 4 + 12*i) + tmpBuff = inMIXFile.read(4) + tmpRdTuple = struct.unpack('I', tmpBuff) + idOfMIXEntry = tmpRdTuple[0] + tmpBuff = inMIXFile.read(4) + tmpRdTuple = struct.unpack('I', tmpBuff) + offsetOfMIXEntry = tmpRdTuple[0] + tmpBuff = inMIXFile.read(4) + tmpRdTuple = struct.unpack('I', tmpBuff) + sizeOfMIXEntry = tmpRdTuple[0] + + for suppTREFileName in supportedExportedTREFiles: + if(idOfMIXEntry == calculateFoldHash(suppTREFileName)): + foundTREFile = True + currTreFileName = suppTREFileName + break + + if (foundTREFile == True): + print "Entry Name: %s, Entry ID: %s, offset %s, data segment %s bytes" % (currTreFileName, ''.join('{:08X}'.format(idOfMIXEntry)), ''.join('{:08X}'.format(offsetOfMIXEntry)),''.join('{:08X}'.format(sizeOfMIXEntry))) + # + # IF TRE FILE: + # put file in TRE object + # + # + inMIXFile.seek(2 + 4 + 12*numOfEntriesToExtract + offsetOfMIXEntry) + if(offsetOfMIXEntry + sizeOfMIXEntry > allMixFileSize): + print "Error: TRE file size mismatch with reported size in entry header!" + else: + treFileBuffer = inMIXFile.read(sizeOfMIXEntry) + if (len(treFileBuffer) == sizeOfMIXEntry): + # load TRE file + thisTreFile = treFile() + if (thisTreFile.loadTreFile(treFileBuffer, allMixFileSize)): + print "TRE file loaded" + if excelOutBook != None: + sh = excelOutBook.add_sheet(currTreFileName) + n = 0 # keeps track of rows + col1_name = 'Text Resource File: %s' % (currTreFileName) + sh.write(n, 0, col1_name) + # Second Row + n = 1 + col1_name = 'TextId' + col2_name = 'Text' + sh.write(n, 0, col1_name) + sh.write(n, 1, col2_name) + n+=1 + for m, e1 in enumerate(thisTreFile.stringEntriesLst, n): + sh.write(m, 0, e1[0]) + objStr = e1[1] + #print type (objUTF8SafeStr) # the type is STR here + # python strings are immutable (can't replace characters) but we have an issue with certain special characters in the ORIGINAL TRE (kiacred and endcred) + # (they are out of their order from their proper order in windwos-1252) + # so we need to create a new string. + objUTF8SafeStr = "" + for i in range(0, len(objStr)): + if (objStr[i] == '\x81'): + objUTF8SafeStr += 'ü' + elif (objStr[i] == '\x82'): + objUTF8SafeStr += 'é' + else: + objUTF8SafeStr += objStr[i] + #objUTF8Safe = objUTF8Safe.replace('\x81',u'u') #'ü' # this does not work + #objUTF8Safe = objUTF8Safe.replace('\x82',u'e') #'é' # this does not work + objUTF8Unicode = unicode(objUTF8SafeStr, 'utf-8') + sh.write(m, 1, objUTF8Unicode) + + + #for tupleIdString in thisTreFile.stringEntriesLst: + # #print "Id: %d\t Text: %s" % (tupleIdString[0], tupleIdString[1]) + # pass + totalTREs = totalTREs + 1 + else: + print "Error while LOADING TRE file!" + else: + print "Error while reading TRE file %s into mem buffer" % (''.join('{:08X}'.format(idOfMIXEntry))) + inMIXFile.close() + print "Total TREs: %d " % (totalTREs) + return + + +# +# Creating the OUTPUT XLS file with one sheet named as the @param sheet with entries based on the list1 (wav files, without duplicates) +# +def outputXLS(filename, sheet, list1, parseTREResourcesAlso = False, mixInputFolderPath = ''): + global stringReplacementForRootFolderWithExtractedFiles + global numReplaceStartingCharacters + book = xlwt.Workbook() + sh = book.add_sheet(sheet) +# First Row + n = 0 # keeps track of rows +# variables = [x, y, z] +# x_desc = 'Display' +# y_desc = 'Dominance' +# z_desc = 'Test' +# desc = [x_desc, y_desc, z_desc] +# +# +# #You may need to group the variables together +# #for n, (v_desc, v) in enumerate(zip(desc, variables)): +# for n, (v_desc, v) in enumerate(zip(desc, variables)): +# sh.write(n, 0, v_desc) +# sh.write(n, 1, v) + col1_name = 'BladeRunnerTLK In-Game dialogue / voiceover quotes' + sh.write(n, 0, col1_name) +# Second Row + n = 1 + col1_name = 'Filename' + col2_name = 'Quote' + col3_name = 'By Actor' + col4_name = 'Notes' + col5_name = 'To Actor' + col6_name = 'Resource' + col7_name = 'ShortHandFileName' + + sh.write(n, 0, col1_name) + sh.write(n, 1, col2_name) + sh.write(n, 2, col3_name) + sh.write(n, 3, col4_name) + sh.write(n, 4, col5_name) + sh.write(n, 5, col6_name) + sh.write(n, 6, col7_name) + + n+=1 + + for m, e1 in enumerate(list1, n): + twoTokensOfRelDirnameAndFilenameXLS = e1.split('&', 2) + if len(twoTokensOfRelDirnameAndFilenameXLS) == 3: + fourTokensOfFilename = twoTokensOfRelDirnameAndFilenameXLS[0].split('#', 3) + if len(fourTokensOfFilename) == 4: + # fix rogue _ chars in 3rd token of filename (split at '_') + tmpAUDFileName = fourTokensOfFilename[0] + '-' + fourTokensOfFilename[1] + '.AUD' + #ActorId-QuoteId.AUD + sh.write(m, 0, tmpAUDFileName) + twoTokensOfQuote = fourTokensOfFilename[2].split('-', 1) + if len(twoTokensOfQuote) == 2: + #Quote + sh.write(m, 1, twoTokensOfQuote[1]) + else: + #Quote + sh.write(m, 1, fourTokensOfFilename[2]) + #Notes + sh.write(m, 3, 'TODO') + #byActor + sh.write(m, 2, fourTokensOfFilename[3]) + #ResourceFolder + sh.write(m, 5, twoTokensOfRelDirnameAndFilenameXLS[1]) + #ShortHandFileName + tmpActorShortHand = getActorShortNameById(fourTokensOfFilename[0]) + shortHandFileName = tmpActorShortHand + '_' + fourTokensOfFilename[1] + '_' + fourTokensOfFilename[2] + '.WAV' + # real path of filename + realPathOfFileNameToLink = twoTokensOfRelDirnameAndFilenameXLS[2] + # checks if not empty + if stringReplacementForRootFolderWithExtractedFiles and numReplaceStartingCharacters > 0: + realPathOfFileNameToLink = realPathOfFileNameToLink.replace(realPathOfFileNameToLink[:numReplaceStartingCharacters], stringReplacementForRootFolderWithExtractedFiles) + + #works in Linux + Libreoffice + # also works in Windows + LibreOffice (run from msys) -- tried something like: + # python sortBladeRunnerWavs.py -p /g/WORKSPACE/BladeRunnerWorkspace/br-mixer-master/data/WAV -m "G:/WORKSPACE/BladeRunnerWorkspace/br-mixer-master/data/WAV" + #TODO put real full path for each file as FILE URL, and real (or approximate shorthand file name as alias) + hyperlinkAudioFormula = 'HYPERLINK("file://%s","%s")' % (realPathOfFileNameToLink, shortHandFileName) + sh.write(m, 6, Formula(hyperlinkAudioFormula)) + else: + sh.write(m, 0, e1) + #Notes + sh.write(m, 3, 'error') + else: + sh.write(m, 0, e1) + #Notes + sh.write(m, 3, 'error') + + + # works for filenames where some rogue greek characters exist + #sh.write(m, 0, str.decode("%s" % e1, 'utf-8')) + +# for m, e2 in enumerate(list2, n+1): +# sh.write(m, 1, e2) + + if parseTREResourcesAlso == True and mixInputFolderPath != '': + inputMIXExtractTREs(mixInputFolderPath, book) + # TODO add sheets + # TODO handle special string characters (to UTF-8) + + book.save(filename) + +# +# +# +# ######################## +# main +# 00_0000 -- DealsInInsects dupl TLK01, TLK0A +# 00_0510 -- ThinkingOfChangingJobs-Leon dupl TLK02, TLK03 +# 00-8520 -- WhatDoYouKnow dupl TLK01, TLK0A + +# Total unique quotes seems to be 5495! +# TODO rename files in folders to conform to the underscore '_' and '-' format (a few don't -- let's have them all conforming!) +# ######################### +# +if __name__ == "__main__": + TMProotFolderWithExtractedFiles = "" + TMProotFolderWithInputTLKFiles = "" + + extractWavFilesMode = False + extractTreFilesMode = False + + invalidSyntax = False +# print "Len of sysargv = %s" % (len(sys.argv)) + if len(sys.argv) == 2: + if(sys.argv[1] == '--help'or sys.argv[1] == '-h'): + print "%s %s supports Blade Runner (English version, CD edition)." % (app_name_spaced, app_version) + print "Created by Praetorian of the classic adventures in Greek team." + print "Always keep backups!" + print "--------------------" + print "Preparatory steps:" + print "1. Put actornames.txt file in the same folder with this tool." + print "--------------------" + print "%s takes has one mandatory argument, the folder of the extracted WAV files:" % (app_name_spaced) + print "Valid syntax: %s -ip [folderpath_for_TLK_Files] -op [folderpath_for_extracted_wav_Files] -m [stringPathToReplaceFolderpathInExcelLinks]" % (app_name) + print "The -op switch has an argument that is the path for extracted WAV files folder. The -op switch is REQUIRED always." + print "The -ip switch has an argument that is the path for the input (TLK or MIX) files folder (can be the same as the Blade Runner installation folder)." + print "The -m switch has an argument that is a replacement string for the path to the folder of extracted WAV files which will be used as a prefix for the links in the output XLS file." + print "The -xwav switch enables the WAV audio extract mode from the TLK files. It requires an INPUT path to be set with the -ip switch." + print "The -xtre switch enables the TRE parsing mode from the original MIX files. It requires an INPUT path to be set with the -ip switch." + print "If the app finishes successfully a sortedWavs.xls file will be created in the same folder with the app." + print "--------------------" + print "Thank you for using this app." + print "Please provide any feedback to: %s " % (company_email) + sys.exit() + elif(sys.argv[1] == '--version' or sys.argv[1] == '-v'): + print "%s %s supports Blade Runner (English version, CD edition)." % (app_name_spaced, app_version) + print "Please provide any feedback to: %s " % (company_email) + sys.exit() + else: + invalidSyntax = True + elif len(sys.argv) > 2: + for i in range(1, len(sys.argv)): + if( i < (len(sys.argv) - 1) and sys.argv[i][:1] == '-' and sys.argv[i+1][:1] != '-'): + if (sys.argv[i] == '-op'): + TMProotFolderWithExtractedFiles = sys.argv[i+1] + numReplaceStartingCharacters = len(TMProotFolderWithExtractedFiles) + elif (sys.argv[i] == '-ip'): + TMProotFolderWithInputTLKFiles = sys.argv[i+1] + elif (sys.argv[i] == '-m'): + stringReplacementForRootFolderWithExtractedFiles = sys.argv[i+1] + elif (sys.argv[i] == '-xwav'): + print "Extract WAVs from TLK files mode enabled." + extractWavFilesMode = True + elif (sys.argv[i] == '-xtre'): + print "Extract TRE mode enabled." + extractTreFilesMode = True + if not TMProotFolderWithExtractedFiles: # this argument is mandatory + invalidSyntax = True + + if (extractWavFilesMode == True or extractTreFilesMode == True) and (TMProotFolderWithInputTLKFiles == ''): + invalidSyntax = True + + if not invalidSyntax: + + # parse Actors files: + initActorPropertyEntries() +# for actorEntryTmp in actorPropertyEntries: +# print "Found actor: %s %s %s" % (actorEntryTmp[0], actorEntryTmp[1], actorEntryTmp[2]) + # + # Checking for the optional case of parsing the input TLK files to extract to WAV + # + if TMProotFolderWithInputTLKFiles != '': + if (extractWavFilesMode == True): + inputTLKsExtract(TMProotFolderWithInputTLKFiles, TMProotFolderWithExtractedFiles) + #if (extractTreFilesMode == True): + # inputMIXExtractTREs(TMProotFolderWithInputTLKFiles) + # + # Parsing the extracted WAV files + # + print "Parsing the extracted WAV audio files. Please wait (it could take a while)..." + for (dirpath, dirnames, filenames) in walk(TMProotFolderWithExtractedFiles): + for nameIdx, nameTmp in enumerate(filenames): + relDirName = '' +# os.path.split would Split the pathname path into a pair, (head, tail) where tail is the last pathname component and head is everything leading up to that. The tail part will never contain a slash + pathTokens = dirpath.split(os.path.sep) + for pTokenTmp in pathTokens: + if pTokenTmp.find("TLK") != -1: + relDirName = pTokenTmp +# print os.path.dirname(dirpath) +# print os.path.abspath(os.path.join(os.path.join(dirpath, nameTmp), os.pardir)) + filenames[nameIdx] = filenames[nameIdx] +'&' + relDirName + '&' + os.path.join(dirpath, nameTmp) + wavfiles.extend(filenames) +# break + for fileIdx, filenameTmp in enumerate(wavfiles): + twoTokensOfFilenameAndRelDirname = filenameTmp.split('&', 1) + if len(twoTokensOfFilenameAndRelDirname) != 2: + print "ERROR in filename and rel dirname split: %s" % (filenameTmp) + sys.exit(0) + twoTokensOfFilenameForExt = twoTokensOfFilenameAndRelDirname[0].split('.', 1) + if len(twoTokensOfFilenameForExt) == 2: + if twoTokensOfFilenameForExt[1] != 'WAV' and twoTokensOfFilenameForExt[1] != 'wav': + print "ERROR in proper extension (not WAV): %s" % (twoTokensOfFilenameAndRelDirname[0]) + sys.exit(0) + else: + print "ERROR in extension split: %s" % (twoTokensOfFilenameAndRelDirname[0]) + sys.exit(0) + #remove WAV extension here +# filenameTmp = twoTokensOfFilenameAndRelDirname[0] + '&' + twoTokensOfFilenameForExt[0] +# print "Found %s" % (filenameTmp) + + threeTokensOfFilename = twoTokensOfFilenameForExt[0].split('_', 2) + if len(threeTokensOfFilename) == 3: + # fix rogue _ chars in 3rd token of filename (split at '_') + threeTokensOfFilename[2] = threeTokensOfFilename[2].replace("_", "-") + # Replace first token + # replace actor name shorthand with ActorID in first part + tmpActorId = getActorIdByShortName(threeTokensOfFilename[0]) + tmpActorFullName = '' + if(tmpActorId != '' and tmpActorId is not None): + tmpActorFullName = getActorFullNameById(tmpActorId) + if(tmpActorFullName != '' and tmpActorFullName is not None): + threeTokensOfFilename[0] = tmpActorId.zfill(2) + threeTokensOfFilename.append(tmpActorFullName) + else: + #fatal error if something cannot convert to spot it immediately + print "ERROR in actorIdMatch match: %s %s" % (tmpActorId, twoTokensOfFilenameForExt[0]) + sys.exit(0) + else: + #fatal error if something cannot convert to spot it immediately + print "ERROR in shorthand match: %s %s" % (threeTokensOfFilename[0], twoTokensOfFilenameForExt[0]) + sys.exit(0) +# +# +# foundMatchForActorShortHand = False +# for actorEntryTmp in actorPropertyEntries: +# if actorEntryTmp[1] == threeTokensOfFilename[0]: +# threeTokensOfFilename[0] = actorEntryTmp[0].zfill(2) +# threeTokensOfFilename.append(actorEntryTmp[2]) +# foundMatchForActorShortHand = True +# break + # end of replace actor name shorthand + twoTokensOfFilenameForExt[0] = '#'.join(threeTokensOfFilename) + filenameTmp = twoTokensOfFilenameForExt[0] + '&' + twoTokensOfFilenameAndRelDirname[1] + wavfiles[fileIdx] = filenameTmp + else: + print "ERROR in spliting tokens on _: %s" % (filenameTmp) + sys.exit(0) + #sort in-place + # + # + wavfiles.sort() +# # +# # +# # Code for renaming non conforming filenames - just to be consistent in file naming +# # TO BE RUN ONCE FOR CONFORMANCE. No NEED TO Re-RUN +# # If its run though, comment this section and re-run the tool to get proper links in Excel file +# # +# for filenameSrcTmp in wavfiles: +# # get real full path from last token when split at & +# # create target full path from the parentdir of last token and the current state of first 3 tokens when splitting at '#' +# # report mismatch +# # print (BUT DON'T PROCEED AT THIS POINT) what you would rename to what. +# threeTokensOfFilenameAndRelDirname = filenameSrcTmp.split('&', 2) +# currentSrcFullPath = threeTokensOfFilenameAndRelDirname[2] +# fourTokensOfTargetFileName = threeTokensOfFilenameAndRelDirname[0].split('#', 3) +# tmpActorShortHand = getActorShortNameById(fourTokensOfTargetFileName[0]) +# targetFileName = tmpActorShortHand + '_' + fourTokensOfTargetFileName[1] + '_' + fourTokensOfTargetFileName[2] + '.WAV' +# # os.path.split would Split the pathname path into a pair, (head, tail) where tail is the last pathname component and head is everything leading up to that. The tail part will never contain a slash +# (srcParentDir, srcTail) = os.path.split(currentSrcFullPath) +# targetFullPath = os.path.join(srcParentDir, targetFileName) +# # os.rename(src, dst) +# if(currentSrcFullPath != targetFullPath): +# print currentSrcFullPath +# print targetFullPath +# os.rename(currentSrcFullPath, targetFullPath) + # + # END OF: Code for renaming non conforming filenames - just to be consistent in file naming + # + # + for filenameSrcTmp in wavfiles: + duplicateFound = False +# print "Converted %s" % (filenameSrcTmp) + # Weed out duplicates by copying to another table (quick and dirty) + twoTokensOfRelDirnameAndFilenameSrc = filenameSrcTmp.split('&', 2) + tmpRelDirNameSrc = twoTokensOfRelDirnameAndFilenameSrc[1] + threeTokensOfQuoteFilenameSrc = twoTokensOfRelDirnameAndFilenameSrc[0].split('#', 2) + #concatenate actorID and quoteID for search key + keyForDuplicateSearchSrc = threeTokensOfQuoteFilenameSrc[0] + threeTokensOfQuoteFilenameSrc[1] + for fileTargIdx, filenameTargTmp in enumerate(wavfilesNoDups): + twoTokensOfRelDirnameAndFilenameTarg = filenameTargTmp.split('&', 2) + tmpRelDirNameTarg = twoTokensOfRelDirnameAndFilenameTarg[1] + threeTokensOfQuoteFilenameTarg = twoTokensOfRelDirnameAndFilenameTarg[0].split('#', 2) + #concatenate actorID and quoteID for search key + keyForDuplicateSearchTarg = threeTokensOfQuoteFilenameTarg[0] + threeTokensOfQuoteFilenameTarg[1] + if(keyForDuplicateSearchSrc == keyForDuplicateSearchTarg): + #print "Found duplicate %s" % (filenameSrcTmp) + duplicateFound = True + wavfilesNoDups[fileTargIdx] = twoTokensOfRelDirnameAndFilenameTarg[0] + '&' + tmpRelDirNameSrc + ',' + tmpRelDirNameTarg + '&' + twoTokensOfRelDirnameAndFilenameTarg[2] + break + if(duplicateFound == False): + wavfilesNoDups.append(filenameSrcTmp) +# for filenameSrcTmp in wavfilesNoDups: +# print "Unique %s" % (filenameSrcTmp) + + print "Creating output excel %s file..." % (OUTPUT_XLS_FILENAME) + outputXLS(OUTPUT_XLS_FILENAME, OUTPUT_XLS_QUOTES_SHEET, wavfilesNoDups, extractTreFilesMode, TMProotFolderWithInputTLKFiles) + else: + invalidSyntax = True + + if invalidSyntax == True: + print "Invalid syntax\n Try: \n %s -op [folderpath_for_extracted_wav_Files] \n %s --help for more info \n %s --version for version info " % (app_name, app_name, app_name) + tmpi = 0 + for tmpArg in sys.argv: + if tmpi==0: #skip first argument + tmpi+=1 + continue + print "\nArgument: %s" % (tmpArg) + tmpi+=1 +else: + ## debug + #print '%s was imported from another module' % (app_name_spaced,) + pass diff --git a/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/treFileLib.py b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/treFileLib.py new file mode 100644 index 0000000000..b9d147248d --- /dev/null +++ b/devtools/blade_runner/subtitles/quotesSpreadsheetCreator/treFileLib.py @@ -0,0 +1,107 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +# +import os, sys, shutil +import struct +from struct import * + +my_module_version = "0.50" +my_module_name = "treFileLib" + + +class TreHeader: + numOfTextResources = -1 + def __init__(self): + return + + +class treFile: + m_header = TreHeader() + stringEntriesLst = [] # list of two-value tuples. First value is ID, second value is String content + stringOffsets = [] + def __init__(self): + del self.stringEntriesLst[:] + del self.stringOffsets[:] + return + + def loadTreFile(self, treBytesBuff, maxLength): + offsInTreFile = 0 + # + # parse TRE file fields for header + # + try: + tmpTuple = struct.unpack_from('I', treBytesBuff, offsInTreFile) # unsigned integer 4 bytes + self.header().numOfTextResources = tmpTuple[0] + offsInTreFile += 4 + # + # string IDs table (each entry is unsigned integer 4 bytes) + # + print "Total texts in TRE: %d" % (self.header().numOfTextResources) + for idx in range(0, self.header().numOfTextResources): + tmpTuple = struct.unpack_from('I', treBytesBuff, offsInTreFile) # unsigned integer 4 bytes + self.stringEntriesLst.append( (tmpTuple[0], '') ) + offsInTreFile += 4 + + # string offsets table (each entry is unsigned integer 4 bytes) + for idx in range(0, self.header().numOfTextResources): + tmpTuple = struct.unpack_from('I', treBytesBuff, offsInTreFile) # unsigned integer 4 bytes + self.stringOffsets.append( tmpTuple[0] ) + offsInTreFile += 4 + # + # strings (all entries are null terminated) + # TODO +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + absStartOfIndexTable = 4 + #absStartOfOffsetTable = absStartOfIndexTable + (self.header().numOfTextResources * 4) + #absStartOfStringTable = absStartOfOffsetTable + ((self.header().numOfTextResources+1) * 4) + + #print "buffer type " , type(treBytesBuff) # it is str + + for idx in range(0, self.header().numOfTextResources): + currOffset = self.stringOffsets[idx] + absStartOfIndexTable + # the buffer (treBytesBuff) where we read the TRE file into, is "str" type but contains multiple null terminated strings + # the solution here (to not get out of index errors when reading the null terminator points) is + # to split the substring starting at the indicated offset each time, at the null character, and get the first string token. + # This works ok. + # + allTextsFound = treBytesBuff[currOffset:].split('\x00') + # check "problematic" character cases: + if currOffset == 5982 or currOffset == 6050 or currOffset == 2827 or currOffset == 2880: + print "Offs: %d\tFound String: %s" % ( currOffset,''.join(allTextsFound[0]) ) + #print "Offs: %d\tFound String: %s" % ( currOffset,''.join(allTextsFound[0]) ) + (theId, stringOfIdx) = self.stringEntriesLst[idx] + self.stringEntriesLst[idx] = (theId, ''.join(allTextsFound[0])) + #print "ID: %d\tFound String: %s" % ( theId,''.join(allTextsFound[0]) ) + return True + except: + print "Loading failure!" + return False + + def header(self): + return self.m_header +# +# +# +if __name__ == '__main__': + # main() + print "Running %s as main module" % (my_module_name) + # assumes a file of name ACTORS.TRE in same directory + inTREFile = None + errorFound = False + try: + inTREFile = open(os.path.join('.','ACTORS.TRE'), 'rb') + except: + errorFound = True + print "Unexpected error:", sys.exc_info()[0] + raise + if not errorFound: + allOfTreFileInBuffer = inTREFile.read() + treFileInstance = treFile() + if (treFileInstance.loadTreFile(allOfTreFileInBuffer, len(allOfTreFileInBuffer))): + print "TRE file loaded successfully!" + else: + print "Error while loading TRE file!" + inTREFile.close() +else: + #debug + #print "Running %s imported from another module" % (my_module_name) + pass
\ No newline at end of file |
