From eb1245a64a69dcbbf96687b43f2211aff2db0598 Mon Sep 17 00:00:00 2001 From: Jamie Hardt Date: Sat, 13 Sep 2025 21:04:14 -0700 Subject: [PATCH] Fixed TOKENIZES_PARALLELISM env setting --- ucsinfer/__main__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ucsinfer/__main__.py b/ucsinfer/__main__.py index 986577c..ccad36a 100644 --- a/ucsinfer/__main__.py +++ b/ucsinfer/__main__.py @@ -39,7 +39,7 @@ def ucsinfer(ctx, verbose, no_model_cache, model, complete_ucs): """ Tools for applying UCS categories to sounds using large-language Models """ - + if verbose: stream_handler.setLevel(logging.DEBUG) logger.info("Verbose logging is enabled") @@ -51,6 +51,10 @@ def ucsinfer(ctx, verbose, no_model_cache, model, complete_ucs): stream_handler.setLevel(logging.WARNING) + os.environ['TOKENIZERS_PARALLELISM'] = 'false' + logger.info("Setting TOKENIZERS_PARALLELISM environment variable to `false" + " explicitly") + ctx.ensure_object(dict) ctx.obj['model_cache'] = not no_model_cache ctx.obj['model_name'] = model @@ -199,7 +203,7 @@ def finetune(ctx): @click.pass_context def evaluate(ctx, dataset, offset, limit): """ - + Evaluate performance of a model """ logger.debug("EVALUATE mode") logger.warning("Model evaluation is not currently implemented") @@ -207,6 +211,4 @@ def evaluate(ctx, dataset, offset, limit): if __name__ == '__main__': - os.environ['TOKENIZERS_PARALLELISM'] = 'false' - ucsinfer(obj={})