Update setup.py

Added package_data

.github/workflows/pythonpublish.yml

@@ -0,0 +1,29 @@
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Install parsimonious
+      run: |
+        pip install parsimonious
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*

.gitignore

@@ -102,3 +102,4 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+.DS_Store

.travis.yml

@@ -1,6 +1,7 @@
 language: python
 python:
   - "3.7"
+  - "3.8
 script:
   - "python -m unittest discover tests"
 install:

README.md

@@ -1,5 +1,8 @@
 [![Build Status](https://travis-ci.com/iluvcapra/ptulsconv.svg?branch=master)](https://travis-ci.com/iluvcapra/ptulsconv)
  ![](https://img.shields.io/github/license/iluvcapra/ptulsconv.svg) ![](https://img.shields.io/pypi/pyversions/ptulsconv.svg) [![](https://img.shields.io/pypi/v/ptulsconv.svg)](https://pypi.org/project/ptulsconv/) ![](https://img.shields.io/pypi/wheel/ptulsconv.svg)
+ 
+ ![Upload Python Package](https://github.com/iluvcapra/ptulsconv/workflows/Upload%20Python%20Package/badge.svg)
+ 
 # ptulsconv
 Read Pro Tools text exports and generate XML, JSON, reports
 
@@ -111,7 +114,3 @@ cues will be applied (later clips having precedence). The clips need not be touc
 into a single row of the output. The start time of the first clip will become the start time of the row, and the finish 
 time of the last clip will become the finish time of the row.
 
-## Other Projects
-
-This project is under construction. Look at [Pro Tools Text](https://github.com/iluvcapra/ProToolsText) 
-for a working solution at this time.

man/ptulsconv.1

@@ -0,0 +1,38 @@
+.\" Manpage for ptulsconv
+.\" Contact https://github.com/iluvcapra/ptulsconv
+.TH ptulsconv 1 "15 May 2020" "0.4.0" "ptulsconv man page"
+.SH NAME
+.BR "ptulsconv" " \- convert 
+.IR "Avid Pro Tools" " text exports"
+.SH SYNOPSIS
+ptulsconv [OPTIONS] Export.txt
+.SH DESCRIPTION
+Convert a Pro Tools text export into a flat list of clip names with timecodes. A tagging
+language is interpreted to add columns and type the data. The default output format is
+an XML file for import into Filemaker Pro.
+.SH OPTIONS
+.IP "-h, --help"
+show a help message and exit.
+.TP 
+.RI "-i " "TC"
+Drop events before this timecode.
+.TP 
+.RI "-o " "TC"
+Drop events after this timecode.
+.TP
+.RI "-m "
+Include muted clips.
+.TP
+.RI "--json "
+Output a JSON document instead of XML. (--xform will have no effect.)
+.TP
+.RI "--xform " "NAME"
+Convert the output with a built-in output transform.
+.TP
+.RI "--show-available-tags"
+Print a list of tags that are interpreted and exit.
+.TP
+.RI "--show-available-transforms"
+Print a list of built-in output transforms and exit.
+.SH AUTHOR
+Jamie Hardt (contact at https://github.com/iluvcapra/ptulsconv)

ptulsconv/__init__.py

@@ -2,6 +2,6 @@ from .ptuls_grammar import protools_text_export_grammar
 from .ptuls_parser_visitor import DictionaryParserVisitor
 from .transformations import TimecodeInterpreter
 
-__version__ = '0.1.0'
+__version__ = '0.4.1'
 __author__  = 'Jamie Hardt'
-__license__ = 'MIT'
+__license__ = 'MIT'

ptulsconv/__main__.py

@@ -1,37 +1,67 @@
-from ptulsconv.commands import convert, dump_field_map
+from ptulsconv.commands import convert, dump_field_map, dump_xform_options
 from ptulsconv import __name__, __version__, __author__
 from optparse import OptionParser, OptionGroup
 from .reporting import print_status_style, print_banner_style, print_section_header_style, print_fatal_error
 import datetime
 import sys
 
+import traceback
+
+
 def main():
     parser = OptionParser()
     parser.usage = "ptulsconv TEXT_EXPORT.txt"
 
-    parser.add_option('-i', dest='in_time', help="Don't output events occurring before this timecode, and offset"
-                                                 " all events relative to this timecode.", metavar='TC')
-    parser.add_option('-o', dest='out_time', help="Don't output events occurring after this timecode.", metavar='TC')
+    filter_opts = OptionGroup(title='Filtering Options', parser=parser)
+
+    filter_opts.add_option('-i', dest='in_time', help="Don't output events occurring before this timecode.",
+                           metavar='TC')
+    filter_opts.add_option('-o', dest='out_time', help="Don't output events occurring after this timecode.",
+                           metavar='TC')
     # parser.add_option('-P', '--progress', default=False, action='store_true', dest='show_progress',
     #                   help='Show progress bar.')
-    parser.add_option('-m', '--include-muted', default=False, action='store_true', dest='include_muted',
-                      help='Read muted clips.')
+    filter_opts.add_option('-m', '--include-muted', default=False, action='store_true', dest='include_muted',
+                           help='Include muted clips.')
 
-    parser.add_option('--show-available-tags', dest='show_tags',
-                      action='store_true',
-                      default=False, help='Display tag mappings for the FMP XML output style and exit.')
+    filter_opts.add_option('-R', '--reel', dest='select_reel', help="Output only events in reel N, and recalculate "
+                                                                    " start times relative to that reel's start time.",
+                           default=None, metavar='N')
+
+    parser.add_option_group(filter_opts)
+
+    output_opts = OptionGroup(title="Output Options", parser=parser)
+    output_opts.add_option('--json', default=False, action='store_true', dest='write_json',
+                           help='Output a JSON document instead of XML. If this option is enabled, --xform will have '
+                                'no effect.')
+
+    output_opts.add_option('--xform', dest='xslt', help="Convert with built-is XSLT transform.",
+                           default=None, metavar='NAME')
+
+    output_opts.add_option('--show-available-tags', dest='show_tags',
+                           action='store_true',
+                           default=False, help='Display tag mappings for the FMP XML output style and exit.')
+
+    output_opts.add_option('--show-available-transforms', dest='show_transforms',
+                           action='store_true',
+                           default=False, help='Display available built-in XSLT transforms.')
+
+    parser.add_option_group(output_opts)
 
     (options, args) = parser.parse_args(sys.argv)
 
-    print_banner_style("%s %s (c) 2019 %s. All rights reserved." % (__name__, __version__, __author__))
+    print_banner_style("%s %s (c) 2020 %s. All rights reserved." % (__name__, __version__, __author__))
 
     print_section_header_style("Startup")
-    print_status_style("This run started %s" % (datetime.datetime.now().isoformat() ) )
+    print_status_style("This run started %s" % (datetime.datetime.now().isoformat()))
 
     if options.show_tags:
         dump_field_map('ADR')
         sys.exit(0)
 
+    if options.show_transforms:
+        dump_xform_options()
+        sys.exit(0)
+
     if len(args) < 2:
         print_fatal_error("Error: No input file")
         parser.print_help(sys.stderr)
@@ -54,8 +84,12 @@ def main():
         print_status_style("Muted regions are ignored.")
 
     try:
-        convert(input_file=args[1], start=options.in_time, end=options.out_time,
-                include_muted=options.include_muted,
+        output_format = 'fmpxml'
+        if options.write_json:
+            output_format = 'json'
+
+        convert(input_file=args[1], output_format=output_format, start=options.in_time, end=options.out_time,
+                include_muted=options.include_muted, xsl=options.xslt, select_reel=options.select_reel,
                 progress=False, output=sys.stdout, log_output=sys.stderr)
     except FileNotFoundError as e:
         print_fatal_error("Error trying to read input file")
@@ -63,6 +97,7 @@ def main():
     except Exception as e:
         print_fatal_error("Error trying to convert file")
         print("\033[31m" + e.__repr__() + "\033[0m", file=sys.stderr)
+        print(traceback.format_exc())
 
 
 if __name__ == "__main__":

ptulsconv/commands.py

@@ -1,8 +1,11 @@
 import io
 import json
+import os
 import os.path
 import sys
 from xml.etree.ElementTree import TreeBuilder, tostring
+import subprocess
+import pathlib
 import ptulsconv
 
 from .reporting import print_section_header_style, print_status_style
@@ -91,6 +94,21 @@ def fmp_dump(data, input_file_name, output):
     output.write(xmlstr)
 
 
+import glob
+
+xslt_path = os.path.join(pathlib.Path(__file__).parent.absolute(), 'xslt')
+
+def xform_options():
+    return glob.glob(os.path.join(xslt_path, "*.xsl"))
+
+def dump_xform_options(output=sys.stdout):
+    print("# Available transforms:", file=output)
+    print("# Transform dir: %s" % (xslt_path), file=output)
+    for f in xform_options():
+        base = os.path.basename(f)
+        name, _ = os.path.splitext(base)
+        print("#    " + name, file=output)
+
 def dump_field_map(field_map_name, output=sys.stdout):
     output.write("# Map of Tag fields to XML output columns\n")
     output.write("# (in order of precedence)\n")
@@ -106,13 +124,28 @@ def dump_field_map(field_map_name, output=sys.stdout):
 
     for n, field in enumerate(field_map):
         for tag in field[0]:
-            output.write("# %-24s-> %-20s | %-8s| %-7i\n" % (tag[:24], field[1][:20], field[2].__name__, n+1 ))
+            output.write("# %-24s-> %-20s | %-8s| %-7i\n" % (tag[:24], field[1][:20], field[2].__name__, n + 1))
 
 
-def convert(input_file, output_format='fmpxml', start=None, end=None,
-            progress=False, include_muted=False,
+def fmp_transformed_dump(data, input_file, xsl_name, output):
+    pipe = io.StringIO()
+    print_status_style("Generating base XML")
+    fmp_dump(data, input_file, pipe)
+
+    strdata = pipe.getvalue()
+    print_status_style("Base XML size %i" % (len(strdata)))
+
+    print_status_style("Running xsltproc")
+
+    xsl_path = os.path.join(pathlib.Path(__file__).parent.absolute(), 'xslt', xsl_name + ".xsl")
+    print_status_style("Using xsl: %s" % (xsl_path))
+    subprocess.run(['xsltproc', xsl_path, '-'], input=strdata, text=True,
+                            stdout=output, shell=False, check=True)
+
+
+def convert(input_file, output_format='fmpxml', start=None, end=None, select_reel=None,
+            progress=False, include_muted=False, xsl=None,
             output=sys.stdout, log_output=sys.stderr):
-
     with open(input_file, 'r') as file:
         print_section_header_style('Parsing')
         ast = ptulsconv.protools_text_export_grammar.parse(file.read())
@@ -143,7 +176,17 @@ def convert(input_file, output_format='fmpxml', start=None, end=None,
             subclipxform = ptulsconv.transformations.SubclipOfSequence(start=start_fs, end=end_fs)
             parsed = subclipxform.transform(parsed)
 
+        if select_reel is not None:
+            reel_xform = ptulsconv.transformations.SelectReel(reel_num=select_reel)
+            parsed = reel_xform.transform(parsed)
+
+
         if output_format == 'json':
             json.dump(parsed, output)
         elif output_format == 'fmpxml':
-            fmp_dump(parsed, input_file, output)
+            if xsl is None:
+                fmp_dump(parsed, input_file, output)
+            else:
+                print_section_header_style("Performing XSL Translation")
+                print_status_style("Using builtin translation: %s" % (xsl))
+                fmp_transformed_dump(parsed, input_file, xsl, output)

ptulsconv/transformations.py

@@ -7,6 +7,7 @@ from .reporting import print_advisory_tagging_error, print_section_header_style,
 
 from tqdm import tqdm
 
+
 class Transformation:
     def transform(self, input_dict) -> dict:
         return input_dict
@@ -163,7 +164,8 @@ class TagInterpreter(Transformation):
                 if clip['state'] == 'Muted' and self.ignore_muted:
                     continue
 
-                clip_tags = self.parse_tags(clip['clip_name'], parent_track_name=track['name'], clip_time=clip['start_time'])
+                clip_tags = self.parse_tags(clip['clip_name'], parent_track_name=track['name'],
+                                            clip_time=clip['start_time'])
                 clip_start = clip['start_time_decoded']['frame_count']
                 if clip_tags['mode'] == 'Normal':
                     event = dict()
@@ -176,6 +178,7 @@ class TagInterpreter(Transformation):
 
                     event['PT.Track.Name'] = track_tags['line']
                     event['PT.Session.Name'] = title_tags['line']
+                    event['PT.Session.TimecodeFormat'] = input_dict['header']['timecode_format']
                     event['PT.Clip.Number'] = clip['event']
                     event['PT.Clip.Name'] = clip_tags['line']
                     event['PT.Clip.Start'] = clip['start_time']
@@ -194,11 +197,14 @@ class TagInterpreter(Transformation):
                     transformed[-1]['PT.Clip.Name'] = transformed[-1]['PT.Clip.Name'] + " " + clip_tags['line']
                     transformed[-1]['PT.Clip.Finish_Frames'] = clip['end_time_decoded']['frame_count']
                     transformed[-1]['PT.Clip.Finish'] = clip['end_time']
-                    transformed[-1]['PT.Clip.Finish_Seconds'] = clip['end_time_decoded']['frame_count'] / input_dict['header'][
-                        'timecode_format']
+                    transformed[-1]['PT.Clip.Finish_Seconds'] = clip['end_time_decoded']['frame_count'] / \
+                                                                input_dict['header']['timecode_format']
+
 
                 elif clip_tags['mode'] == 'Timespan':
-                    rule = dict(start_time=clip_start,
+                    rule = dict(start_time_literal=clip['start_time'],
+                                start_time=clip_start,
+                                start_time_seconds=clip_start / input_dict['header']['timecode_format'],
                                 end_time=clip['end_time_decoded']['frame_count'],
                                 tags=clip_tags['tags'])
                     timespan_rules.append(rule)
@@ -211,7 +217,17 @@ class TagInterpreter(Transformation):
 
         for rule in rules:
             if rule['start_time'] <= time <= rule['end_time']:
+                tag_keys = list(rule['tags'].keys())
+                tag_times = dict()
+                for key in tag_keys:
+                    key: str
+                    time_value = rule['start_time']
+                    tag_times["Timespan." + key + ".Start_Frames"] = time_value
+                    tag_times["Timespan." + key + ".Start"] = rule['start_time_literal']
+                    tag_times["Timespan." + key + ".Start_Seconds"] = rule['start_time_seconds']
+
                 retval.update(rule['tags'])
+                retval.update(tag_times)
 
         return retval
 
@@ -244,6 +260,26 @@ class TagInterpreter(Transformation):
             return self.visitor.visit(parse_tree)
 
 
+class SelectReel(Transformation):
+
+    def __init__(self, reel_num):
+        self.reel_num = reel_num
+
+    def transform(self, input_dict) -> dict:
+        out_events = []
+        for event in input_dict['events']:
+            if event['Reel'] == str(self.reel_num):
+                offset = event.get('Timespan.Reel.Start_Frames', 0)
+                offset_sec = event.get('Timespan.Reel.Start_Seconds', 0.)
+                event['PT.Clip.Start_Frames'] -= offset
+                event['PT.Clip.Finish_Frames'] -= offset
+                event['PT.Clip.Start_Seconds'] -= offset_sec
+                event['PT.Clip.Finish_Seconds'] -= offset_sec
+                out_events.append(event)
+
+        return dict(header=input_dict['header'], events=out_events)
+
+
 class SubclipOfSequence(Transformation):
 
     def __init__(self, start, end):
@@ -252,8 +288,8 @@ class SubclipOfSequence(Transformation):
 
     def transform(self, input_dict: dict) -> dict:
         out_events = []
-        offset = self.start
-        offset_sec = self.start / input_dict['header']['timecode_format']
+        offset = 0 #self.start
+        offset_sec = 0. #self.start / input_dict['header']['timecode_format']
         for event in input_dict['events']:
             if self.start <= event['PT.Clip.Start_Frames'] <= self.end:
                 e = event
@@ -263,4 +299,4 @@ class SubclipOfSequence(Transformation):
                 e['PT.Clip.Finish_Seconds'] = event['PT.Clip.Finish_Seconds'] - offset_sec
                 out_events.append(e)
 
-        return dict(events=out_events)
+        return dict(header=input_dict['header'], events=out_events)

ptulsconv/xslt/AvidMarkers.xsl

@@ -37,8 +37,16 @@
       <AvProp id="ATTR" name="OMFI:ATTB:Kind" type="int32">2</AvProp>
       <AvProp id="ATTR" name="OMFI:ATTB:Name" type="string">_ATN_CRM_COM</AvProp>
       <AvProp id="ATTR" name="OMFI:ATTB:StringAttribute" type="string">
-	<xsl:value-of select="concat(fmp:COL[15]/fmp:DATA, ': ', fmp:COL[21]/fmp:DATA)"/>
-	[Reason: <xsl:value-of select="fmp:COL[18]/fmp:DATA" />]</AvProp>
+	<xsl:value-of select="concat('(',fmp:COL[14]/fmp:DATA,') ',fmp:COL[15]/fmp:DATA, ': ', fmp:COL[21]/fmp:DATA, ' ')"/>
+        <xsl:choose>
+          <xsl:when test="fmp:COL[18]/fmp:DATA != ''">[Reason: <xsl:value-of select="fmp:COL[18]/fmp:DATA" />]
+          </xsl:when>
+          <xsl:otherwise> </xsl:otherwise>
+        </xsl:choose>
+        <xsl:choose>
+          <xsl:when test="fmp:COL[23]/fmp:DATA != ''">[Note: <xsl:value-of select="fmp:COL[23]/fmp:DATA" />]</xsl:when>
+        </xsl:choose>
+      </AvProp>
     </ListElem>
     <ListElem>
       <AvProp id="ATTR" name="OMFI:ATTB:Kind" type="int32">2</AvProp>

ptulsconv/xslt/SRT.xsl

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:transform version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+               xmlns:fmp="http://www.filemaker.com/fmpxmlresult">
+
+<xsl:output method="text" encoding="windows-1252"/>
+<xsl:template match="/">
+
+<xsl:for-each select="/fmp:FMPXMLRESULT/fmp:RESULTSET/fmp:ROW">
+	<xsl:sort data-type="number" select="number(fmp:COL[9]/fmp:DATA)" />
+<xsl:value-of select="concat(position() ,'&#xA;')" />
+	<xsl:value-of select="concat(format-number(floor(number(fmp:COL[9]/fmp:DATA) div 3600),'00'), ':')" />
+	<xsl:value-of select="concat(format-number(floor(number(fmp:COL[9]/fmp:DATA) div 60),'00'), ':')" />
+	<xsl:value-of select="concat(format-number(floor(number(fmp:COL[9]/fmp:DATA) mod 60),'00'), ',')" />
+	<xsl:value-of select="format-number((number(fmp:COL[9]/fmp:DATA) - floor(number(fmp:COL[9]/fmp:DATA))) * 1000,'000')" />
+	<xsl:text> --> </xsl:text>
+
+	<xsl:value-of select="concat(format-number(floor(number(fmp:COL[10]/fmp:DATA) div 3600),'00'), ':')" />
+	<xsl:value-of select="concat(format-number(floor(number(fmp:COL[10]/fmp:DATA) div 60),'00'), ':')" />
+	<xsl:value-of select="concat(format-number(floor(number(fmp:COL[10]/fmp:DATA) mod 60),'00'), ',')" />
+	<xsl:value-of select="format-number((number(fmp:COL[10]/fmp:DATA) - floor(number(fmp:COL[10]/fmp:DATA))) * 1000,'000')" />
+
+	<xsl:value-of select="concat('&#xA;',fmp:COL[15]/fmp:DATA, ': ', fmp:COL[21]/fmp:DATA)"/>
+	<xsl:value-of select="'&#xA;&#xA;'" />
+
+</xsl:for-each>
+</xsl:template>
+
+
+</xsl:transform>

pypi_upload.sh

@@ -1,4 +0,0 @@
-#!/bin/zsh
-python3 setup.py build
-python3 setup.py sdist bdist_wheel
-python3 -m twine upload --repository-url https://upload.pypi.org/legacy/ dist/*

setup.py

@@ -24,15 +24,19 @@ setup(name='ptulsconv',
           'Topic :: Multimedia',
           'Topic :: Multimedia :: Sound/Audio',
           "Programming Language :: Python :: 3.7",
+          "Programming Language :: Python :: 3.8",
           "Development Status :: 4 - Beta",
           "Topic :: Text Processing :: Filters",
           "Topic :: Text Processing :: Markup :: XML"],
       packages=['ptulsconv'],
       keywords='text-processing parsers film tv editing editorial',
-      install_requires=['parsimonious','tqdm'],
+      install_requires=['parsimonious', 'tqdm'],
+      package_data={
+          "ptulsconv": ["*.xsl"]
+      },
       entry_points={
           'console_scripts': [
               'ptulsconv = ptulsconv.__main__:main'
           ]
       }
-      )
+      )

tests/test_tagging.py

@@ -6,6 +6,31 @@ class TaggingIntegratedTests(unittest.TestCase):
 
     path = os.path.dirname(__file__) + '/export_cases/Tag Tests/Tag Tests.txt'
 
+    def test_event_list(self):
+        with open(self.path, 'r') as f:
+            visitor = ptulsconv.DictionaryParserVisitor()
+            result = ptulsconv.protools_text_export_grammar.parse(f.read())
+            parsed: dict = visitor.visit(result)
+
+            tcxform = ptulsconv.transformations.TimecodeInterpreter()
+            tagxform = ptulsconv.transformations.TagInterpreter(show_progress=False,
+                                                                ignore_muted=True,
+                                                                log_output=False)
+
+            parsed = tcxform.transform(parsed)
+            parsed = tagxform.transform(parsed)
+
+            self.assertEqual(9, len(parsed['events']))
+            self.assertEqual("Clip Name", parsed['events'][0]['PT.Clip.Name'])
+            self.assertEqual("Lorem ipsum" , parsed['events'][1]['PT.Clip.Name'])
+            self.assertEqual("Dolor sic amet the rain in spain" , parsed['events'][2]['PT.Clip.Name'])
+            self.assertEqual("A B C" , parsed['events'][3]['PT.Clip.Name'])
+            self.assertEqual("Silver Bridge" , parsed['events'][4]['PT.Clip.Name'])
+            self.assertEqual("Region 02" , parsed['events'][5]['PT.Clip.Name'])
+            self.assertEqual("Region 12" , parsed['events'][6]['PT.Clip.Name'])
+            self.assertEqual("Region 22" , parsed['events'][7]['PT.Clip.Name'])
+            self.assertEqual("Region 04" , parsed['events'][8]['PT.Clip.Name'])
+
     def test_append(self):
         with open(self.path, 'r') as f:
             visitor = ptulsconv.DictionaryParserVisitor()