(OjW*SrSSKJr SSKrSSKJrJr SSKJr SSK J r J r J r J r SSKrSSKJr SSKJr SS KJrJr \R.S - r\R.S - r\R.S - r\R.S - r\"SS55r"SS5rg)a import_utils.py Utility module for importing and validating LeafDoctor AI datasets. Responsibilities ---------------- 1. Read CSV files 2. Clean data 3. Validate data 4. Import into PostgreSQL 5. Generate crop-label mapping 6. Generate import report Author: LeafDoctor AI ) annotationsN) dataclassfield)Path)DictListOptionalSet)Session)settings)ProtectionLabelTreatmentMappingzcrop_protection_labels.csvz&normalized_class_treatment_mapping.csvzcrop_label_mapping.jsonzimport_report.jsonc\rSrSr%SrS\S'SrS\S'SrS\S'SrS\S'Sr S\S'Sr S\S 'Sr S\S 'Sr S\S 'Sr S\S '\"\S 9rS\S'\"\S 9rS\S'SrSrg) ImportReport=rint master_labelstreatment_recordsimported_labelsimported_treatmentsduplicate_labelsduplicate_class_idsorphan_class_idslabel_mismatches skipped_rows)default_factoryz List[dict]warningsz List[str]errorsc URURURURURUR UR URURURURS. $)N rrrrrrrrrrrr selfs ,/var/www/html/leaf-doctor-v1/import_utils.pyto_dictImportReport.to_dictVsm"//!%!7!7#33#'#;#; $ 5 5#'#;#; $ 5 5 $ 5 5 -- kk  N)__name__ __module__ __qualname____firstlineno__r__annotations__rrrrrrrrrlistrrr$__static_attributes__r'r&r#rr=sM3sOS  c  ccL# 6Hj6d3FI3 r&rc\rSrSrSSjrSr\S5r\S5rSr Sr SSjr S r S r S rS rS rSrSrSrSrSrSrSrSrSrSrg) ImportUtilslcfXl[5UlSUlSUl0Ul0UlgN)dbrreport labels_df treatment_dfmaster_label_mapcrop_label_map)r"r4s r#__init__ImportUtils.__init__ns/"n 15480235r&c[R5(d[[5e[R5(d[[5e[R "[5Ul[R "[5Ul[S5 [UR R5 [S5 [URR5 [S5 [URR55 [UR 5URl [UR5URlg)Nz ===== LABELS DTYPES =====z ===== TREATMENT DTYPES =====z ===== TREATMENT HEAD =====) LABEL_FILEexistsFileNotFoundErrorTREATMENT_FILEpdread_csvr6r7printdtypesheadlenr5rrr!s r#load_csv_filesImportUtils.load_csv_filess  ""#J/ /$$&&#N3 3Z0KK7 +, dnn##$ ./ d&&' ,- d$$&'$'$7 !(+   )  %r&c[R"U5(ag[U5nUR5nUS:XagSR UR 55nU$)N )rAisnastrstripjoinsplitvalues r# clean_stringImportUtils.clean_stringsK 775>>E   B;' r&cr[R"U5(ag[U5$![a gf=fr3)rArLr ExceptionrQs r# clean_integerImportUtils.clean_integers5 775>> u:   s ) 66c0URRH8nURURUR5URU'M: URSRUR5URS'UR RH@nUS:XaM UR URUR5UR U'MB UR SRUR5UR S'g)NClass_ID)r6columnsapplyrSrWr7)r"columns r#clean_dataframesImportUtils.clean_dataframess nn,,Fv&t(() NN6 "- NN: & U4%% & z"''//F#!!&)t(()   f % 0   j ) U4%% & *%r&cURR5 URR5H"upUSnUSnUcMX@RU'M$ g)NrZ Label_Name)r8clearr6iterrows)r"_rowclass_idlabels r#build_master_lookupImportUtils.build_master_lookupsW ##%nn--/FA:H %E.3 ! !( +0r&c8URRU5$r3)r8get)r"rfs r# master_labelImportUtils.master_labels$$((22r&cn[5 [S5 [S5 [S5 [5 g)NF======================================================================zLeafDoctor AI Dataset ImportrCr!s r# print_headerImportUtils.print_headers#  h ,- h r&cn[5 [S5 [S5 [S5 [5 g)NrozDataset Loading Finishedrpr!s r# print_footerImportUtils.print_footers#  h () h r&c([S5 URURSRSS9RS5nUR(dfUSR 5UR lUR5H0up#UR RRSUS35 M2 URURSRSS9RS5nUR(dfUSR 5UR l UR5H0up#UR RRSUS35 M2 URURSR5nUR(dYUR =R[U5- sl UR RRS[U535 URURSR5nUR(dZUR =R[U5- sl UR RRS [U535 g g ) z Validate master label dataset. zValidating master labels...rZF)keepzDuplicate Class_ID : razDuplicate Label : zMissing Class_ID rows : zMissing Label rows : N)rCr6 duplicated sort_valuesemptynuniquer5rrcrappendrrLrrF)r" duplicate_idsrdrer missing_idsmissing_labelss r#validate_master_labels"ImportUtils.validate_master_labelss9 +, NNz*5555A [ $ ""j)113 KK +(002 ""))+C O+<=3 NN|,77U7C [ &  %%!.668 KK (+335 ""))(\):(;<6nn NN: & + + -   KK $ $K(8 8 $ KK   % %*3{+;*<=  NN< ( - - / ## KK $ $N(; ; $ KK   % %'N(;'<=  $r&c[S5 [5nURR5GH{up#USnUR S5nUcJUR =R S- slUR RRSUS35 MiX@R;aCURU5 UR RR[U5USS .5 Uc2UR RR[U5US S .5 UR S 5c2UR RR[U5US S .5 UR S 5bGMIUR RR[U5USS .5 GM~ [U5UR lg)z% Validate treatment mapping dataset. zValidating treatment mapping...rZCropNzRow z: Missing Class_IDzOrphan Class_ID)rerfissuez Missing CropChemical & FormulationzMissing ChemicalSMS : Growth Stage 1z Missing SMS)rCsetr7rcrkr5rrr|r8addrrrFr)r" orphan_idsindexrerfcrops r#validate_treatment_mapping&ImportUtils.validate_treatment_mappingos /0U ++446JE:H776?D ((A-( ""))5'!34444x( $$++"5z$,!2| $$++"5z$,!/ww/08 $$++"5z$,!3ww-.6 $$++"5z$,!.e7t(+: $r&c[S5 SnURR5Hup#USnUcMX@R;aM URUnUR UR S55nUcMTXV:wdM[US- nUR RR[U5UUUS.5 M XR l g)zA Compare the label in treatment CSV with the master label table. zChecking label mapping...rrZNMaster_Label_Namer)rerfrl csv_label) rCr7rcr8rSrkr5rr|rr)r" mismatchesrrerfrlrs r#detect_mapping_mismatches%ImportUtils.detect_mapping_mismatchess )* ++446JE:H44400:L))+,I (a  $$++"5z$,(4%. /7@(2 $r&cUR5 UR5 UR5 [5 [S5 [SURR 35 [SURR 35 [SURR35 [SURR35 [SURR35 [5 g)z Run every validation step. zValidation CompleteDuplicate Labels : Duplicate Class IDs : Orphan Class IDs : Label Mismatches : Skipped Rows : N) rrrrCr5rrrrrr!s r#validateImportUtils.validates ##% '') &&(  #$ &{{++, . &{{../ 1 &{{++, . &{{++, . &{{''( * r&cV[S5 URR[5R 5 URR[ 5R 5 URR 5 g![a URR5 ef=f)zl Clear master tables before importing. Import is idempotent, so every execution starts from a clean state. zClearing existing data...N) rCr4queryrdeleter commitrVrollbackr!s r#clear_databaseImportUtils.clear_databasesu )*  GGMM* + 2 2 4 GGMM/ * 1 1 3 GGNN   GG      s A4B&B(c v[S5 SnURR5HUup#USnUSnUcMUcM[UUSUR S5SS9nUR R U5 US- nMW UR R5 XRl [S US 35 g) z Import master label table. zImporting protection labels...rrZraNPrimary_Category)rf label_name_en label_name_bnprimary_category_enprimary_category_bnr Imported z labels.) rCr6rcr rkr4rflushr5r)r"importedrdrerfrgobjs r#import_protection_labels$ImportUtils.import_protection_labels2s ./nn--/FA:H %E}!!#"$'GG&%%)C GGKK  MHA0D  &. # (8,-r&c[S5 SnSnURR5Hup4USnUcUS- nMXPR;aUS- nM*[ UUR S5SUR S5SUR S5UR S 5UR S 5UR S 5SS 9 nUR RU5 UR R5 US- nM [[W55 UR R5 XRl UR=RU- sl [S US35 [SUS35 g)z Import treatment mapping. IMPORTANT We intentionally ignore Master_Label_Name from the CSV because the master table already contains the authoritative label. zImporting treatment mapping...rrZNrrzType of Problemrz Chemical TypezRevised Dosagesr) rfcrop_encrop_bnproblem_type_enproblem_type_bnchemical chemical_typedosagesms_ensms_bnrz treatments.zSkipped z rows.) rCr7rcr8rrkr4rrvarsr5rr)r"rskippedrdrerf treatments r#import_treatment_mapping$ImportUtils.import_treatment_mappinggsd ./''002FA:H1 4441 (! #%!!%,"gg#ww%ww*= ID GGKK " GGMMO MHo3p d9o  *2 '   G+  (<01  ()r&c[5 [S5 [S5 [S5 UR5 UR5 UR5 URR 5 [5 [S5 g![ aHnURR5 [5 [S5 [[U55eSnAff=f)z) Complete import inside one transaction. z<============================================================zDatabase Import StartedzDatabase Import Completed.zDatabase Import FailedN) rCrrrr4rrVr RuntimeErrorrM)r"exs r#import_databaseImportUtils.import_databases  h '( h (    !  ) ) +  ) ) + GGNN  G . / ( GG    G * +s2w' ' (sAB CACCc[S5 0nURR5Hup#USnX@R;aMUR UR S55nUcM@URUnXQ;a[ 5[ 5S.X'XSRU5 XSRU5 M 0n[UR55H$upX[US5[US5S.Xu'M& [[SS S 9n [R"UU S S S 9 SSS5 [S[35 g!,(df  N!=f)z~ Generate crop -> valid labels mapping. Example { "Rice": { "labels": [...], "class_ids": [...] } } z Generating crop-label mapping...rZrN)labels class_idsrrwutf-8encodingFindent ensure_asciizCrop mapping saved to )rCr7rcr8rSrkrrsorteditemsopenCROP_MAPPING_FILEjsondump) r"mappingrdrerfrrgoutputvaluesfps r#generate_crop_label_mapping'ImportUtils.generate_crop_label_mappingsW 01''002FA:H444$$SWWV_5D|))(3E""e!$! M( # ' ' . M+ & * *8 4/32"7==?3LD!!12#F;$78 FL4    II"     $ ! #   s E Ec[[SSS9n[R"URR 5USSS9 SSS5 [ S[35 g!,(df  N!=f)NrrrrFrzImport report saved to )rIMPORT_REPORT_FILErrr5r$rC)r"rs r#save_import_reportImportUtils.save_import_report5se     II ##%"     %!" $   s 0A A)c[5 [S5 [S5 [S5 [5 [SURR35 [SURR35 [SURR35 [SURR 35 [SURR 35 [SURR35 [S URR35 [S URR35 [S URR35 [S [URR535 [S [URR535 [5 [S5 [5 g)NrozLeafDoctor AI Import SummaryzMaster Labels : zTreatment Records : zImported Labels : zImported Treatments : rrrrrzWarnings : zErrors : )rCr5rrrrrrrrrrFrrr!s r# print_summaryImportUtils.print_summaryMs  h ,- h  &{{(() + &{{,,- / &{{**+ - &{{../ 1 &{{++, . &{{../ 1 &{{++, . &{{++, . &{{''( * &4;;''() + &4;;%%&' )  h r&cDUR5 UR5 UR5 UR5 UR 5 UR 5 UR 5 UR5 UR5 UR5 g)z Complete import workflow. N) rqrGr^rhrrrrrrtr!s r# run_importImportUtils.run_importsy      "   ((* !  r&)r9r4r6r8r5r7N)r4r )rfr)r(r)r*r+r:rG staticmethodrSrWr^rhrlrqrtrrrrrrrrrrrrr.r'r&r#r0r0ls 6$ <   # R 4&3  SrC7R*2`(\8/.jR*p#(RG Z 0GZr&r0)__doc__ __future__rr dataclassesrrpathlibrtypingrrr r pandasrAsqlalchemy.ormr configr modelsr r data_pathr=r@rrrr0r'r&r#rs$# (,,"  "> >  ,-     & &  & \E E r&