Merge branch 'master' of https://github.com/iluvcapra/ptulsconv

command tweaks

.travis.yml

@@ -0,0 +1,8 @@
+language: python
+python:
+  - "3.7"
+script:
+  - "python -m unittest discover tests"
+install:
+  - "pip install setuptools"
+  - "pip install parsimonious tqdm"

README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.com/iluvcapra/ptulsconv.svg?branch=master)](https://travis-ci.com/iluvcapra/ptulsconv)
+ ![](https://img.shields.io/github/license/iluvcapra/ptulsconv.svg) ![](https://img.shields.io/pypi/pyversions/ptulsconv.svg) [![](https://img.shields.io/pypi/v/ptulsconv.svg)](https://pypi.org/project/ptulsconv/) ![](https://img.shields.io/pypi/wheel/ptulsconv.svg)
 # ptulsconv
 Read Pro Tools text exports and generate XML, JSON, reports
 

ptulsconv/__init__.py

@@ -2,6 +2,6 @@ from .ptuls_grammar import protools_text_export_grammar
 from .ptuls_parser_visitor import DictionaryParserVisitor
 from .transformations import TimecodeInterpreter
 
-__version__ = '0.0.1'
+__version__ = '0.1.0'
 __author__  = 'Jamie Hardt'
 __license__ = 'MIT'

ptulsconv/__main__.py

@@ -1,36 +1,68 @@
 from ptulsconv.commands import convert, dump_field_map
-from optparse import OptionParser
+from ptulsconv import __name__, __version__, __author__
+from optparse import OptionParser, OptionGroup
+from .reporting import print_status_style, print_banner_style, print_section_header_style, print_fatal_error
+import datetime
 import sys
 
-
 def main():
     parser = OptionParser()
     parser.usage = "ptulsconv TEXT_EXPORT.txt"
+
     parser.add_option('-i', dest='in_time', help="Don't output events occurring before this timecode, and offset"
                                                  " all events relative to this timecode.", metavar='TC')
     parser.add_option('-o', dest='out_time', help="Don't output events occurring after this timecode.", metavar='TC')
-    parser.add_option('-P', '--progress', default=False, action='store_true', dest='show_progress',
+    # parser.add_option('-P', '--progress', default=False, action='store_true', dest='show_progress',
-                      help='Show progress bar.')
+    #                   help='Show progress bar.')
     parser.add_option('-m', '--include-muted', default=False, action='store_true', dest='include_muted',
                       help='Read muted clips.')
 
-    parser.add_option('--show-tags', dest='show_tags',
+    parser.add_option('--show-available-tags', dest='show_tags',
                       action='store_true',
                       default=False, help='Display tag mappings for the FMP XML output style and exit.')
 
     (options, args) = parser.parse_args(sys.argv)
 
+    print_banner_style("%s %s (c) 2019 %s. All rights reserved." % (__name__, __version__, __author__))
+
+    print_section_header_style("Startup")
+    print_status_style("This run started %s" % (datetime.datetime.now().isoformat() ) )
+
     if options.show_tags:
         dump_field_map('ADR')
         sys.exit(0)
 
     if len(args) < 2:
-        print("Error: No input file", file=sys.stderr)
+        print_fatal_error("Error: No input file")
         parser.print_help(sys.stderr)
         sys.exit(22)
 
-    convert(input_file=args[1], start=options.in_time, end=options.out_time, include_muted=options.include_muted,
+    print_status_style("Input file is %s" % (args[1]))
-            progress=options.show_progress, output=sys.stdout)
+    if options.in_time:
+        print_status_style("Start at time %s" % (options.in_time))
+    else:
+        print_status_style("No start time given.")
+
+    if options.out_time:
+        print_status_style("End at time %s." % (options.out_time))
+    else:
+        print_status_style("No end time given.")
+
+    if options.include_muted:
+        print_status_style("Muted regions are included.")
+    else:
+        print_status_style("Muted regions are ignored.")
+
+    try:
+        convert(input_file=args[1], start=options.in_time, end=options.out_time,
+                include_muted=options.include_muted,
+                progress=False, output=sys.stdout, log_output=sys.stderr)
+    except FileNotFoundError as e:
+        print_fatal_error("Error trying to read input file")
+        raise e
+    except Exception as e:
+        print_fatal_error("Error trying to convert file")
+        print("\033[31m" + e.__repr__() + "\033[0m", file=sys.stderr)
 
 
 if __name__ == "__main__":

ptulsconv/commands.py

@@ -5,6 +5,8 @@ import sys
 from xml.etree.ElementTree import TreeBuilder, tostring
 import ptulsconv
 
+from .reporting import print_section_header_style, print_status_style
+
 # field_map maps tags in the text export to fields in FMPXMLRESULT
 #  - tuple field 0 is a list of tags, the first tag with contents will be used as source
 #  - tuple field 1 is the field in FMPXMLRESULT
@@ -107,17 +109,27 @@ def dump_field_map(field_map_name, output=sys.stdout):
             output.write("# %-24s-> %-20s | %-8s| %-7i\n" % (tag[:24], field[1][:20], field[2].__name__, n+1 ))
 
 
-def convert(input_file, output_format='fmpxml', start=None, end=None, progress=False, include_muted=False,
+def convert(input_file, output_format='fmpxml', start=None, end=None,
-            output=sys.stdout):
+            progress=False, include_muted=False,
+            output=sys.stdout, log_output=sys.stderr):
+
     with open(input_file, 'r') as file:
+        print_section_header_style('Parsing')
         ast = ptulsconv.protools_text_export_grammar.parse(file.read())
         dict_parser = ptulsconv.DictionaryParserVisitor()
         parsed = dict_parser.visit(ast)
 
-        tcxform = ptulsconv.transformations.TimecodeInterpreter()
+        print_status_style('Session title: %s' % parsed['header']['session_name'])
-        tagxform = ptulsconv.transformations.TagInterpreter(show_progress=progress, ignore_muted=(not include_muted))
+        print_status_style('Session timecode format: %f' % parsed['header']['timecode_format'])
+        print_status_style('Fount %i tracks' % len(parsed['tracks']))
+        print_status_style('Found %i markers' % len(parsed['markers']))
 
-        parsed = tagxform.transform(tcxform.transform(parsed))
+        tcxform = ptulsconv.transformations.TimecodeInterpreter()
+        tagxform = ptulsconv.transformations.TagInterpreter(show_progress=progress, ignore_muted=(not include_muted),
+                                                            log_output=log_output)
+
+        parsed = tcxform.transform(parsed)
+        parsed = tagxform.transform(parsed)
 
         if start is not None and end is not None:
             start_fs = tcxform.convert_time(start,

ptulsconv/ptuls_grammar.py

@@ -45,7 +45,7 @@ protools_text_export_grammar = Grammar(
     
     track_state_list     = (track_state " ")*
     
-    track_state          = "Solo" / "Muted" / "Inactive"
+    track_state          = "Solo" / "Muted" / "Inactive" / "Hidden"
     
     track_clip_entry     = integer_value isp fs 
                            integer_value isp fs 

ptulsconv/reporting.py

@@ -0,0 +1,51 @@
+import sys
+
+def print_banner_style(str):
+    if sys.stderr.isatty():
+        sys.stderr.write("\n\033[1m%s\033[0m\n\n" % str)
+    else:
+        sys.stderr.write("\n%s\n\n" % str)
+
+def print_section_header_style(str):
+    if sys.stderr.isatty():
+        sys.stderr.write("\n\033[4m%s\033[0m\n\n" % str)
+    else:
+        sys.stderr.write("%s\n\n" % str)
+
+def print_status_style(str):
+    if sys.stderr.isatty():
+        sys.stderr.write("\033[3m - %s\033[0m\n" % str)
+    else:
+        sys.stderr.write(" - %s\n" % str)
+
+def print_advisory_tagging_error(failed_string, position, parent_track_name=None, clip_time=None):
+    if sys.stderr.isatty():
+        sys.stderr.write("\n")
+        sys.stderr.write(" ! \033[33;1mTagging error: \033[0m")
+        ok_string = failed_string[:position]
+        not_ok_string = failed_string[position:]
+        sys.stderr.write("\033[32m\"%s\033[31;1m%s\"\033[0m\n" % (ok_string, not_ok_string))
+
+        if parent_track_name is not None:
+            sys.stderr.write(" !   > On track \"%s\"\n" % (parent_track_name))
+
+        if clip_time is not None:
+            sys.stderr.write(" !   > In clip name at %s\n" % (clip_time))
+    else:
+        sys.stderr.write("\n")
+        sys.stderr.write(" ! Tagging error: \"%s\"\n" % failed_string)
+        sys.stderr.write(" ! %s _______________⬆\n" % (" " * position))
+
+        if parent_track_name is not None:
+            sys.stderr.write(" !   > On track \"%s\"\n" % (parent_track_name))
+
+        if clip_time is not None:
+            sys.stderr.write(" !   > In clip name at %s\n" % (clip_time))
+
+    sys.stderr.write("\n")
+
+def print_fatal_error(str):
+    if sys.stderr.isatty():
+        sys.stderr.write("\n\033[5;31;1m*** %s ***\033[0m\n" % str)
+    else:
+        sys.stderr.write("\n%s\n" % str)

ptulsconv/transformations.py

@@ -3,6 +3,7 @@ from parsimonious import Grammar, NodeVisitor
 from parsimonious.exceptions import IncompleteParseError
 import math
 import sys
+from .reporting import print_advisory_tagging_error, print_section_header_style, print_status_style
 
 from tqdm import tqdm
 
@@ -17,19 +18,27 @@ class TimecodeInterpreter(Transformation):
         self.apply_session_start = False
 
     def transform(self, input_dict: dict) -> dict:
+        print_section_header_style('Converting Timecodes')
+
         retval = super().transform(input_dict)
         rate = input_dict['header']['timecode_format']
         start_tc = self.convert_time(input_dict['header']['start_timecode'], rate,
                                      drop_frame=input_dict['header']['timecode_drop_frame'])
 
         retval['header']['start_timecode_decoded'] = start_tc
+        print_status_style('Converted start timecode.')
+
         retval['tracks'] = self.convert_tracks(input_dict['tracks'], timecode_rate=rate,
                                                drop_frame=retval['header']['timecode_drop_frame'])
 
+        print_status_style('Converted clip timecodes for %i tracks.' % len(retval['tracks']))
+
         for marker in retval['markers']:
             marker['location_decoded'] = self.convert_time(marker['location'], rate,
                                                            drop_frame=retval['header']['timecode_drop_frame'])
 
+        print_status_style('Converted %i markers.' % len(retval['markers']))
+
         return retval
 
     def convert_tracks(self, tracks, timecode_rate, drop_frame):
@@ -121,16 +130,19 @@ class TagInterpreter(Transformation):
         def generic_visit(self, node, visited_children):
             return visited_children or node
 
-    def __init__(self, ignore_muted=True, show_progress=False):
+    def __init__(self, ignore_muted=True, show_progress=False, log_output=sys.stderr):
         self.visitor = TagInterpreter.TagListVisitor()
         self.ignore_muted = ignore_muted
         self.show_progress = show_progress
+        self.log_output = log_output
 
     def transform(self, input_dict: dict) -> dict:
         transformed = list()
         timespan_rules = list()
 
-        title_tags = self.parse_tags(input_dict['header']['session_name'], "<Session Name>")
+        print_section_header_style('Parsing Tags')
+
+        title_tags = self.parse_tags(input_dict['header']['session_name'])
         markers = sorted(input_dict['markers'], key=lambda m: m['location_decoded']['frame_count'])
 
         if self.show_progress:
@@ -142,8 +154,8 @@ class TagInterpreter(Transformation):
             if 'Muted' in track['state'] and self.ignore_muted:
                 continue
 
-            track_tags = self.parse_tags(track['name'], "<Track %s>" % (track['name']))
+            track_tags = self.parse_tags(track['name'], parent_track_name=track['name'])
-            comment_tags = self.parse_tags(track['comments'], "<Track %s>" % (track['name']))
+            comment_tags = self.parse_tags(track['comments'], parent_track_name=track['name'])
             track_context_tags = track_tags['tags']
             track_context_tags.update(comment_tags['tags'])
 
@@ -151,8 +163,7 @@ class TagInterpreter(Transformation):
                 if clip['state'] == 'Muted' and self.ignore_muted:
                     continue
 
-                clip_tags = self.parse_tags(clip['clip_name'],
+                clip_tags = self.parse_tags(clip['clip_name'], parent_track_name=track['name'], clip_time=clip['start_time'])
-                                            "<Track %s/Clip event number %i at %s>" % (track['name'], clip['event'], clip['start_time']))
                 clip_start = clip['start_time_decoded']['frame_count']
                 if clip_tags['mode'] == 'Normal':
                     event = dict()
@@ -180,8 +191,11 @@ class TagInterpreter(Transformation):
                     assert len(transformed) > 0, "First clip is in '&'-Append mode, fatal error."
 
                     transformed[-1].update(clip_tags['tags'])
-                    transformed[-1]['event_name'] = transformed[-1]['event_name'] + " " + clip_tags['line']
+                    transformed[-1]['PT.Clip.Name'] = transformed[-1]['PT.Clip.Name'] + " " + clip_tags['line']
-                    transformed[-1]['PT.Clip.End_Frames'] = clip['end_time_decoded']['frame_count']
+                    transformed[-1]['PT.Clip.Finish_Frames'] = clip['end_time_decoded']['frame_count']
+                    transformed[-1]['PT.Clip.Finish'] = clip['end_time']
+                    transformed[-1]['PT.Clip.Finish_Seconds'] = clip['end_time_decoded']['frame_count'] / input_dict['header'][
+                        'timecode_format']
 
                 elif clip_tags['mode'] == 'Timespan':
                     rule = dict(start_time=clip_start,
@@ -189,6 +203,7 @@ class TagInterpreter(Transformation):
                                 tags=clip_tags['tags'])
                     timespan_rules.append(rule)
 
+        print_status_style('Processed %i clips' % len(transformed))
         return dict(header=input_dict['header'], events=transformed)
 
     def effective_timespan_tags_at_time(_, rules, time) -> dict:
@@ -204,8 +219,8 @@ class TagInterpreter(Transformation):
         retval = dict()
 
         for marker in markers:
-            marker_name_tags = self.parse_tags(marker['name'], "Marker %i" % (marker['number']))
+            marker_name_tags = self.parse_tags(marker['name'], marker_index=marker['number'])
-            marker_comment_tags = self.parse_tags(marker['comments'], "Marker %i" % (marker['number']))
+            marker_comment_tags = self.parse_tags(marker['comments'], marker_index=marker['number'])
             effective_tags = marker_name_tags['tags']
             effective_tags.update(marker_comment_tags['tags'])
 
@@ -215,26 +230,20 @@ class TagInterpreter(Transformation):
                 break
         return retval
 
-    def report(self, mesg, *args):
+    def parse_tags(self, source, parent_track_name=None, clip_time=None, marker_index=None):
-        print(mesg % ( args) , file=sys.stderr)
-        sys.stderr.write("\033[F")
-        sys.stderr.write("\033[K")
-
-    def parse_tags(self, source, context_str=None):
         try:
             parse_tree = self.tag_grammar.parse(source)
             return self.visitor.visit(parse_tree)
         except IncompleteParseError as e:
-            if context_str is not None:
+            print_advisory_tagging_error(failed_string=source,
-                self.report("Error reading tags in: ")
+                                         parent_track_name=parent_track_name,
+                                         clip_time=clip_time, position=e.pos)
 
             trimmed_source = source[:e.pos]
             parse_tree = self.tag_grammar.parse(trimmed_source)
             return self.visitor.visit(parse_tree)
 
 
-
-
 class SubclipOfSequence(Transformation):
 
     def __init__(self, start, end):

setup.py

@@ -29,7 +29,7 @@ setup(name='ptulsconv',
           "Topic :: Text Processing :: Markup :: XML"],
       packages=['ptulsconv'],
       keywords='text-processing parsers film tv editing editorial',
-      install_requires=['parsimonious'],
+      install_requires=['parsimonious','tqdm'],
       entry_points={
           'console_scripts': [
               'ptulsconv = ptulsconv.__main__:main'

tests/export_cases/Tag Tests/Tag Tests.txt

@@ -0,0 +1,100 @@
+SESSION NAME:	Tag Tests
+SAMPLE RATE:	48000.000000
+BIT DEPTH:	24-bit
+SESSION START TIMECODE:	01:00:00:00
+TIMECODE FORMAT:	23.976 Frame
+# OF AUDIO TRACKS:	8
+# OF AUDIO CLIPS:	0
+# OF AUDIO FILES:	0
+
+
+P L U G - I N S  L I S T I N G
+MANUFACTURER            	PLUG-IN NAME            	VERSION         	FORMAT          	STEMS                   	NUMBER OF INSTANCES
+
+
+T R A C K  L I S T I N G
+TRACK NAME:	Audio 1
+COMMENTS:	
+USER DELAY:	0 Samples
+STATE: 
+PLUG-INS: 
+CHANNEL 	EVENT   	CLIP NAME                     	START TIME    	END TIME      	DURATION      	STATE
+1       	1       	Clip Name {X=300}             	   01:00:00:00	   01:00:05:03	   00:00:05:03	Unmuted
+
+
+TRACK NAME:	Audio 2 $A=1
+COMMENTS:	
+USER DELAY:	0 Samples
+STATE: 
+PLUG-INS: 
+CHANNEL 	EVENT   	CLIP NAME                     	START TIME    	END TIME      	DURATION      	STATE
+1       	1       	Lorem ipsum {X=301}           	   01:00:00:00	   01:00:05:03	   00:00:05:03	Unmuted
+1       	2       	Dolor sic amet {X=302}        	   01:00:10:00	   01:00:20:00	   00:00:10:00	Unmuted
+1       	3       	& the rain in spain [ABC]     	   01:00:20:00	   01:00:25:00	   00:00:05:00	Unmuted
+
+
+TRACK NAME:	Audio 3 $A=2
+COMMENTS:	{B=100}
+USER DELAY:	0 Samples
+STATE: 
+PLUG-INS: 
+CHANNEL 	EVENT   	CLIP NAME                     	START TIME    	END TIME      	DURATION      	STATE
+1       	1       	A                             	   01:00:15:00	   01:00:25:00	   00:00:10:00	Unmuted
+1       	2       	& B                           	   01:00:25:00	   01:00:35:00	   00:00:10:00	Unmuted
+1       	3       	& C                           	   01:00:35:00	   01:00:45:00	   00:00:10:00	Unmuted
+
+
+TRACK NAME:	Audio 4 $A=3
+COMMENTS:	$A=4
+USER DELAY:	0 Samples
+STATE: 
+PLUG-INS: 
+CHANNEL 	EVENT   	CLIP NAME                     	START TIME    	END TIME      	DURATION      	STATE
+1       	1       	Silver Bridge                 	   01:00:00:00	   01:00:05:00	   00:00:05:00	Unmuted
+
+
+TRACK NAME:	Audio 5
+COMMENTS:	
+USER DELAY:	0 Samples
+STATE: 
+PLUG-INS: 
+CHANNEL 	EVENT   	CLIP NAME                     	START TIME    	END TIME      	DURATION      	STATE
+1       	1       	@ {D=100}                     	   01:00:00:00	   01:00:10:00	   00:00:10:00	Unmuted
+1       	2       	@ {D=101}                     	   01:00:10:00	   01:00:20:00	   00:00:10:00	Unmuted
+1       	3       	@ {D=102}                     	   01:00:20:00	   01:00:30:00	   00:00:10:00	Unmuted
+
+
+TRACK NAME:	Audio 6
+COMMENTS:	
+USER DELAY:	0 Samples
+STATE: 
+PLUG-INS: 
+CHANNEL 	EVENT   	CLIP NAME                     	START TIME    	END TIME      	DURATION      	STATE
+1       	1       	Region 02                     	   01:00:02:00	   01:00:03:00	   00:00:01:00	Unmuted
+1       	2       	Region 12                     	   01:00:12:00	   01:00:13:00	   00:00:01:00	Unmuted
+1       	3       	Region 22                     	   01:00:22:00	   01:00:23:00	   00:00:01:00	Unmuted
+
+
+TRACK NAME:	Audio 7
+COMMENTS:	
+USER DELAY:	0 Samples
+STATE: 
+PLUG-INS: 
+CHANNEL 	EVENT   	CLIP NAME                     	START TIME    	END TIME      	DURATION      	STATE
+1       	1       	@ {D=200} {E=101}             	   01:00:00:00	   01:00:10:00	   00:00:10:00	Unmuted
+
+
+TRACK NAME:	Audio 8
+COMMENTS:	
+USER DELAY:	0 Samples
+STATE: 
+PLUG-INS: 
+CHANNEL 	EVENT   	CLIP NAME                     	START TIME    	END TIME      	DURATION      	STATE
+1       	1       	Region 04                     	   01:00:04:00	   01:00:05:00	   00:00:01:00	Unmuted
+
+
+M A R K E R S  L I S T I N G
+#   	LOCATION     	TIME REFERENCE    	UNITS    	NAME                             	COMMENTS
+3   	01:00:05:00  	240240            	Samples  	Marker $M=0                      	
+1   	01:00:10:00  	480480            	Samples  	$M=1                             	
+2   	01:00:22:00  	1057056           	Samples  	$M=2                             	

tests/test_tagging.py

@@ -0,0 +1,67 @@
+import unittest
+import ptulsconv
+import os.path
+
+class TaggingIntegratedTests(unittest.TestCase):
+
+    path = os.path.dirname(__file__) + '/export_cases/Tag Tests/Tag Tests.txt'
+
+    def test_append(self):
+        with open(self.path, 'r') as f:
+            visitor = ptulsconv.DictionaryParserVisitor()
+            result = ptulsconv.protools_text_export_grammar.parse(f.read())
+            parsed: dict = visitor.visit(result)
+
+            tcxform = ptulsconv.transformations.TimecodeInterpreter()
+            tagxform = ptulsconv.transformations.TagInterpreter(show_progress=False,
+                                                                ignore_muted=True,
+                                                                log_output=False)
+
+            parsed = tcxform.transform(parsed)
+            parsed = tagxform.transform(parsed)
+
+            self.assertTrue(len(parsed['events']) > 2)
+
+            self.assertEqual("Dolor sic amet the rain in spain",
+                             parsed['events'][2]['PT.Clip.Name'])
+
+            self.assertTrue("01:00:10:00", parsed['events'][2]['PT.Clip.Start'])
+            self.assertTrue("01:00:25:00", parsed['events'][2]['PT.Clip.Finish'])
+            self.assertTrue(240, parsed['events'][2]['PT.Clip.Start_Frames'])
+            self.assertTrue(600, parsed['events'][2]['PT.Clip.Finish_Frames'])
+
+            self.assertIn('X', parsed['events'][2].keys())
+            self.assertIn('ABC', parsed['events'][2].keys())
+            self.assertIn('A', parsed['events'][2].keys())
+            self.assertEqual('302', parsed['events'][2]['X'])
+            self.assertEqual('ABC', parsed['events'][2]['ABC'])
+            self.assertEqual('1', parsed['events'][2]['A'])
+
+    def test_successive_appends(self):
+        with open(self.path, 'r') as f:
+            visitor = ptulsconv.DictionaryParserVisitor()
+            result = ptulsconv.protools_text_export_grammar.parse(f.read())
+            parsed: dict = visitor.visit(result)
+
+            tcxform = ptulsconv.transformations.TimecodeInterpreter()
+            tagxform = ptulsconv.transformations.TagInterpreter(show_progress=False,
+                                                                ignore_muted=True,
+                                                                log_output=False)
+
+            parsed = tcxform.transform(parsed)
+            parsed = tagxform.transform(parsed)
+
+            self.assertTrue(len(parsed['events']) > 3)
+
+            self.assertEqual("A B C",
+                             parsed['events'][3]['PT.Clip.Name'])
+
+            self.assertTrue("01:00:15:00", parsed['events'][3]['PT.Clip.Start'])
+            self.assertTrue("01:00:45:00", parsed['events'][3]['PT.Clip.Finish'])
+            self.assertTrue(80, parsed['events'][3]['PT.Clip.Start_Frames'])
+            self.assertTrue(1080, parsed['events'][3]['PT.Clip.Finish_Frames'])
+
+
+
+if __name__ == '__main__':
+    unittest.main()