Move python scripts into tasks

Makefile

@@ -15,7 +15,7 @@ $(PRE_COMMIT_HOOK):
 	poetry run pre-commit install --install-hooks
 
 $(LOCAL_MODEL):
-	poetry run python converter/huggingface_gptneox_convert.py \
+	poetry run python tasks/converter/huggingface_gptneox_convert.py \
 		-in_file EleutherAI/pythia-70m-deduped \
 		-o $@ \
 		-i_g 1 -m_n tiny-70M -p 1 -w fp16

README.md

@@ -12,8 +12,7 @@ An opensource / on-prem alternative to GitHub Copilot
 * [`admin`](./admin): Admin panel for monitoring / settings purpose.
 * [`server`](./server): API server for completion requests. It also logs users' selections (as feedback to model's quality).
 * [`deployment`](./deployment): Container related deployment configs.
-* [`converter`](./converter): Converts a [transformers](https://huggingface.co/docs/transformers) causal LM model into TensorRT / FasterTransformer serving formats.
-* [`preprocess`](./preprocess): Preprocess files into [datasets](https://huggingface.co/docs/datasets)
+* [`tasks`](./tasks): Various data processing scripts.
 * [`tabformer`](./tabformer): *NOT RELEASED* Trainer(PEFT w/RLHF) for tabby models.
 
 ## Development

tasks/README.md

@@ -0,0 +1,2 @@
+* [`converter`](./converter): Converts a [transformers](https://huggingface.co/docs/transformers) causal LM model into TensorRT / FasterTransformer serving formats.
+* [`preprocess`](./preprocess): Preprocess files into [datasets](https://huggingface.co/docs/datasets)