jamie/ucsinfer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Fixed TOKENIZES_PARALLELISM env setting

Browse Source
This commit is contained in:
Jamie Hardt
2025-09-13 21:04:14 -07:00
parent 615d8ab279
commit eb1245a64a
1 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
ucsinfer/__main__.py
View File

@@ -39,7 +39,7 @@ def ucsinfer(ctx, verbose, no_model_cache, model, complete_ucs):
""" """
Tools for applying UCS categories to sounds using large-language Models Tools for applying UCS categories to sounds using large-language Models
""" """
if verbose: if verbose:
stream_handler.setLevel(logging.DEBUG) stream_handler.setLevel(logging.DEBUG)
logger.info("Verbose logging is enabled") logger.info("Verbose logging is enabled")
@@ -51,6 +51,10 @@ def ucsinfer(ctx, verbose, no_model_cache, model, complete_ucs):
stream_handler.setLevel(logging.WARNING) stream_handler.setLevel(logging.WARNING)
os.environ['TOKENIZERS_PARALLELISM'] = 'false'
logger.info("Setting TOKENIZERS_PARALLELISM environment variable to `false"
" explicitly")
ctx.ensure_object(dict) ctx.ensure_object(dict)
ctx.obj['model_cache'] = not no_model_cache ctx.obj['model_cache'] = not no_model_cache
ctx.obj['model_name'] = model ctx.obj['model_name'] = model
@@ -199,7 +203,7 @@ def finetune(ctx):
@click.pass_context @click.pass_context
def evaluate(ctx, dataset, offset, limit): def evaluate(ctx, dataset, offset, limit):
""" """
Evaluate performance of a model
""" """
logger.debug("EVALUATE mode") logger.debug("EVALUATE mode")
logger.warning("Model evaluation is not currently implemented") logger.warning("Model evaluation is not currently implemented")
@@ -207,6 +211,4 @@ def evaluate(ctx, dataset, offset, limit):
if __name__ == '__main__': if __name__ == '__main__':
os.environ['TOKENIZERS_PARALLELISM'] = 'false'
ucsinfer(obj={}) ucsinfer(obj={})