From c9eef9d6ca30d1dce38de2ee2c3d76ef8a68e197 Mon Sep 17 00:00:00 2001 From: Jamie Hardt Date: Wed, 9 Oct 2019 16:45:48 -0700 Subject: [PATCH 1/6] Fixed some bad parsing behavior in a certain case --- ptulsconv/ptuls_grammar.py | 4 ++-- ptulsconv/ptuls_parser_visitor.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ptulsconv/ptuls_grammar.py b/ptulsconv/ptuls_grammar.py index 200d3bc..72afa67 100644 --- a/ptulsconv/ptuls_grammar.py +++ b/ptulsconv/ptuls_grammar.py @@ -14,13 +14,13 @@ protools_text_export_grammar = Grammar( files_section = files_header files_column_header file_record* block_ending files_header = "F I L E S I N S E S S I O N" rs - files_column_header = "Filename " fs "Location" rs + files_column_header = "Filename" isp fs "Location" rs file_record = string_value fs string_value rs clips_section = clips_header clips_column_header clip_record* block_ending clips_header = "O N L I N E C L I P S I N S E S S I O N" rs clips_column_header = string_value fs string_value rs - clip_record = string_value fs string_value fs "[" integer_value "]" rs + clip_record = string_value fs string_value (fs "[" integer_value "]")? rs plugin_listing = plugin_header plugin_column_header plugin_record* block_ending plugin_header = "P L U G - I N S L I S T I N G" rs diff --git a/ptulsconv/ptuls_parser_visitor.py b/ptulsconv/ptuls_parser_visitor.py index 5136497..0e55a63 100644 --- a/ptulsconv/ptuls_parser_visitor.py +++ b/ptulsconv/ptuls_parser_visitor.py @@ -40,7 +40,9 @@ class DictionaryParserVisitor(NodeVisitor): @staticmethod def visit_clips_section(node, visited_children): - return list(map(lambda child: dict(clip_name=child[0], file=child[2], channel=child[5]), + channel = next(iter(visited_children[2][5]), 1) + + return list(map(lambda child: dict(clip_name=child[0], file=child[2], channel=channel), visited_children[2])) @staticmethod From b2fcc5c215f964cb66e8c964d612554c44607ead Mon Sep 17 00:00:00 2001 From: Jamie Hardt Date: Wed, 9 Oct 2019 16:54:59 -0700 Subject: [PATCH 2/6] Rewrote this so entry_point works --- ptulsconv/__main__.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ptulsconv/__main__.py b/ptulsconv/__main__.py index ecc2bff..296a031 100644 --- a/ptulsconv/__main__.py +++ b/ptulsconv/__main__.py @@ -2,13 +2,13 @@ from ptulsconv.commands import convert from optparse import OptionParser import sys -parser = OptionParser() -parser.usage = "ptulsconv TEXT_EXPORT.txt" -parser.add_option('-i', dest='in_time', help='Set in time to grab a subsequence of events. ' - 'Give value as a timecode in current session\'s rate.') -parser.add_option('-o', dest='out_time', help='Set out time to grab a subsequence of events.') +def main(): + parser = OptionParser() + parser.usage = "ptulsconv TEXT_EXPORT.txt" + parser.add_option('-i', dest='in_time', help='Set in time to grab a subsequence of events. ' + 'Give value as a timecode in current session\'s rate.') + parser.add_option('-o', dest='out_time', help='Set out time to grab a subsequence of events.') -if __name__ == "__main__": (options, args) = parser.parse_args(sys.argv) if len(args) < 2: print("Error: No input file",file=sys.stderr) @@ -16,3 +16,7 @@ if __name__ == "__main__": exit(-1) convert(input_file=args[1], start=options.in_time, end=options.out_time, output=sys.stdout) + +if __name__ == "__main__": + main() + From fdb025482ca68b067bb3280160899f191f5aba15 Mon Sep 17 00:00:00 2001 From: Jamie Hardt Date: Wed, 9 Oct 2019 17:13:21 -0700 Subject: [PATCH 3/6] README --- README.md | 114 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1d8d958..6ccb656 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,115 @@ # ptulsconv -Read Pro Tools Text exports and generate XML, JSON, reports +Read Pro Tools text exports and generate XML, JSON, reports -This projoect is under construction. Look at [Pro Tools Text](https://github.com/iluvcapra/ProToolsText) +## Quick Example + +At this time we're using `ptulsconv` mostly for converting ADR notes in a Pro Tools session +into an XML document we can import into Filemaker Pro. + + % ptulsconv STAR_WARS_IV_R1_ADR_Notes_PT_Text_Export.txt > SW4_r1_ADR_Notes.xml + % xmllint --format SW4_r1_ADR_Notes.xml + + + 0 + + + + + + + + + + + [... much much more] + +## Installation + +The easiest way to install on your site is to use `pip`: + + % pip3 install ptulsconv + +This will install the necessary libraries on your host and gives you command-line access to the tool through an +entry-point `ptulsconv`. In a terminal window type `ptulsconv -h` for a list of available options. + +## Theory of Operation + +[Avid Pro Tools][avp] exports a tab-delimited text file organized in multiple parts with an uneven syntax that usually +can't "drop in" to other tools like Excel or Filemaker. This tool accepts a text export from Pro Tools and produces an +XML file in the `FMPXMLRESULT` schema which Filemaker Pro can import directly into a new table. + +In the default mode, all of the clips are parsed and converted into a flat list of events, one Filemaker Pro row per +clip with a start and finish time, track name, session name, etc. Timecodes are parsed and converted into frame counts +and seconds. Text is then parsed for descriptive meta-tags and these are assigned to columns in the output list. + +[avp]: http://www.avid.com/pro-tools + +### Fields in Clip Names + +Track names, track comments, and clip names can also contain meta-tags, or "fields," to add additional columns to the +output. Thus, if a clip has the name: + +`Fireworks explosion {note=Replace for final} $V=1 [FX] [DESIGN]` + +The row output for this clip will contain columns for the values: + +|...| PT.Clip.Name| note | V | FX | DESIGN | ...| +|---|------------|------|---|----|--------|----| +|...| Fireworks explosion| Replace for final | 1 | FX | DESIGN | ... | + +These fields can be defined in the clip name in three ways: +* `$NAME=VALUE` creates a field named `NAME` with a one-word value `VALUE`. +* `{NAME=VALUE}` creates a field named `NAME` with the value `VALUE`. `VALUE` in this case may contain spaces or any chartacter +up to the closing bracket. +* `[NAME]` creates a field named `NAME` with a value `NAME`. This can be used to create a boolean-valued field; in the CSV +output, clips with the field will have it, and clips without will have the column with an empty value. + +For example, if two clips are named: + +`"Squad fifty-one, what is your status?" [FUTZ] {Ch=Dispatcher} [ADR]` + +`"We are ten-eight at Rampart Hospital." {Ch=Gage} [ADR]` + +The output will contain the range: + +|...| PT.Clip.Name| Ch | FUTZ | ADR | ...| +|---|------------|------|---|----|-----| +|...| "Squad fifty-one, what is your status?"| Dispatcher | FUTZ | ADR | ... | +|...| "We are ten-eight at Rampart Hospital."| Gage | | ADR | ... | + + +### Fields in Track Names and Markers + +Fields set in track names, and in track comments, will be applied to *each* clip on that track. If a track comment +contains the text `{Dept=Foley}` for example, every clip on that track will have a "Foley" value in a "Dept" column. + +Likewise, fields set on the session name will apply to all clips in the session. + +Fields set in markers, and in marker comments, will be applied to all clips whose finish is *after* that marker. Fields +in markers are applied cumulatively from breakfast to dinner in the session. The latest marker applying to a clip has +precedence, so if one marker comes after the other, but both define a field, the value in the later marker + +An important note here is that, always, fields set on the clip name have the highest precedence. If a field is set in a clip +name, the same field set on the track, the value set on the clip will prevail. + +### Using `@` to Apply Fields to a Span of Clips + +A clip name beginning with "@" will not be included in the CSV output, but its fields will be applied to clips within +its time range on lower tracks. + +If track 1 has a clip named `@ {Sc=1- The House}`, any clips beginning within that range on lower tracks will have a +field `Sc` with that value. + +### Using `&` to Combine Clips + +A clip name beginning with `&` will have its parsed clip name appended to the preceding cue, and the fields of following +cues will be applied (later clips having precedence). The clips need not be touching, and the clips will be combined +into a single row of the output. The start time of the first clip will become the start time of the row, and the finish +time of the last clip will become the finish time of the row. + +## Other Projects + +This project is under construction. Look at [Pro Tools Text](https://github.com/iluvcapra/ProToolsText) for a working solution at this time. From 7565766f29dd5cbc0785f25dab0c6f49ead40b8c Mon Sep 17 00:00:00 2001 From: Jamie Hardt Date: Wed, 9 Oct 2019 17:29:22 -0700 Subject: [PATCH 4/6] typos --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6ccb656..113a32e 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,10 @@ The row output for this clip will contain columns for the values: These fields can be defined in the clip name in three ways: * `$NAME=VALUE` creates a field named `NAME` with a one-word value `VALUE`. -* `{NAME=VALUE}` creates a field named `NAME` with the value `VALUE`. `VALUE` in this case may contain spaces or any chartacter -up to the closing bracket. -* `[NAME]` creates a field named `NAME` with a value `NAME`. This can be used to create a boolean-valued field; in the CSV -output, clips with the field will have it, and clips without will have the column with an empty value. +* `{NAME=VALUE}` creates a field named `NAME` with the value `VALUE`. `VALUE` in this case may contain spaces or any + character up to the closing bracket. +* `[NAME]` creates a field named `NAME` with a value `NAME`. This can be used to create a boolean-valued field; in the + output, clips with the field will have it, and clips without will have the column with an empty value. For example, if two clips are named: @@ -104,7 +104,7 @@ field `Sc` with that value. ### Using `&` to Combine Clips -A clip name beginning with `&` will have its parsed clip name appended to the preceding cue, and the fields of following +A clip name beginning with "&" will have its parsed clip name appended to the preceding cue, and the fields of following cues will be applied (later clips having precedence). The clips need not be touching, and the clips will be combined into a single row of the output. The start time of the first clip will become the start time of the row, and the finish time of the last clip will become the finish time of the row. From 615adff575256141043cd27635c4f9c5fcc84e0b Mon Sep 17 00:00:00 2001 From: Jamie Hardt Date: Wed, 9 Oct 2019 17:33:52 -0700 Subject: [PATCH 5/6] Making all these markers one frame long The Avid won't let span markers overlap so they aren't very useful. --- ptulsconv/xslt/FMPXML_AvidMarkers.xsl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ptulsconv/xslt/FMPXML_AvidMarkers.xsl b/ptulsconv/xslt/FMPXML_AvidMarkers.xsl index 6875ce6..7b4fa18 100644 --- a/ptulsconv/xslt/FMPXML_AvidMarkers.xsl +++ b/ptulsconv/xslt/FMPXML_AvidMarkers.xsl @@ -54,10 +54,7 @@ 1 _ATN_CRM_LENGTH - - - - + 1 From dc9e68a2346aaaa57e2fbdfce0920cc517082090 Mon Sep 17 00:00:00 2001 From: Jamie Hardt Date: Wed, 9 Oct 2019 18:04:54 -0700 Subject: [PATCH 6/6] Working on a 'dump' function to help users format their tags --- ptulsconv/commands.py | 47 +++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/ptulsconv/commands.py b/ptulsconv/commands.py index 5a4e78b..bd23b41 100644 --- a/ptulsconv/commands.py +++ b/ptulsconv/commands.py @@ -1,19 +1,15 @@ +import io import json import os.path import sys from xml.etree.ElementTree import TreeBuilder, tostring import ptulsconv -from tqdm import tqdm - -def fmp_dump(data, input_file_name, output): - doc = TreeBuilder(element_factory=None) - - # field_map maps tags in the text export to fields in FMPXMLRESULT - # - tuple field 0 is a list of tags, the first tag with contents will be used as source - # - tuple field 1 is the field in FMPXMLRESULT - # - tuple field 2 the constructor/type of the field - field_map = ((['Title', 'PT.Session.Name'], 'Title', str), +# field_map maps tags in the text export to fields in FMPXMLRESULT +# - tuple field 0 is a list of tags, the first tag with contents will be used as source +# - tuple field 1 is the field in FMPXMLRESULT +# - tuple field 2 the constructor/type of the field +adr_field_map = ((['Title', 'PT.Session.Name'], 'Title', str), (['Supv'], 'Supervisor', str), (['Client'], 'Client', str), (['Sc'], 'Scene', str), @@ -44,6 +40,9 @@ def fmp_dump(data, input_file_name, output): (['ADLIB'], 'Adlib', str), (['OPT'], 'Optional', str)) +def fmp_dump(data, input_file_name, output): + doc = TreeBuilder(element_factory=None) + doc.start('FMPXMLRESULT', {'xmlns': 'http://www.filemaker.com/fmpxmlresult'}) doc.start('ERRORCODE') @@ -58,24 +57,20 @@ def fmp_dump(data, input_file_name, output): doc.end('DATABASE') doc.start('METADATA') - for field in field_map: + for field in adr_field_map: tp = field[2] ft = 'TEXT' if tp is int or tp is float: ft = 'NUMBER' - doc.start('FIELD', {'EMPTYOK': 'YES', - 'MAXREPEAT': '1', - 'NAME': field[1], - 'TYPE': ft}) - + doc.start('FIELD', {'EMPTYOK': 'YES', 'MAXREPEAT': '1', 'NAME': field[1], 'TYPE': ft}) doc.end('FIELD') doc.end('METADATA') doc.start('RESULTSET', {'FOUND': str(len(data['events']))}) for event in data['events']: doc.start('ROW') - for field in field_map: + for field in adr_field_map: doc.start('COL') doc.start('DATA') for key_attempt in field[0]: @@ -93,6 +88,24 @@ def fmp_dump(data, input_file_name, output): output.write(xmlstr) +def dump_field_map(field_map_name, output=sys.stdout): + output.write("# Map of Tag fields to XML output columns\n") + output.write("# (in order of precedence)\n") + output.write("# \n") + field_map = [] + if field_map_name == 'ADR': + field_map = adr_field_map + output.write("# ADR Table Fields\n") + + output.write("# \n") + output.write("# Tag Name | FMPXMLRESULT Column | Type \n") + output.write("# -------------------------+----------------------+---------\n") + + for field in field_map: + for tag in field[0]: + output.write("# %25s| %20s | %8s\n" % (tag[:25], field[1][:20], field[2].__name__)) + + def convert(input_file, output_format='fmpxml', start=None, end=None, output=sys.stdout): with open(input_file, 'r') as file: ast = ptulsconv.protools_text_export_grammar.parse(file.read())