This is page 6 of 19. Use http://codebase.md/mljar/mljar-supervised?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .github
│ └── workflows
│ ├── run-tests.yml
│ ├── test-installation-with-conda.yml
│ └── test-installation-with-pip-on-windows.yml
├── .gitignore
├── CITATION
├── examples
│ ├── notebooks
│ │ ├── basic_run.ipynb
│ │ └── Titanic.ipynb
│ └── scripts
│ ├── binary_classifier_adult_fairness.py
│ ├── binary_classifier_ensemble.py
│ ├── binary_classifier_marketing.py
│ ├── binary_classifier_random.py
│ ├── binary_classifier_Titanic.py
│ ├── binary_classifier.py
│ ├── multi_class_classifier_digits.py
│ ├── multi_class_classifier_MNIST.py
│ ├── multi_class_classifier.py
│ ├── multi_class_drug_fairness.py
│ ├── regression_acs_fairness.py
│ ├── regression_crime_fairness.py
│ ├── regression_housing_fairness.py
│ ├── regression_law_school_fairness.py
│ ├── regression.py
│ └── tabular_mar_2021.py
├── LICENSE
├── MANIFEST.in
├── pytest.ini
├── README.md
├── requirements_dev.txt
├── requirements.txt
├── setup.py
├── supervised
│ ├── __init__.py
│ ├── algorithms
│ │ ├── __init__.py
│ │ ├── algorithm.py
│ │ ├── baseline.py
│ │ ├── catboost.py
│ │ ├── decision_tree.py
│ │ ├── extra_trees.py
│ │ ├── factory.py
│ │ ├── knn.py
│ │ ├── lightgbm.py
│ │ ├── linear.py
│ │ ├── nn.py
│ │ ├── random_forest.py
│ │ ├── registry.py
│ │ ├── sklearn.py
│ │ └── xgboost.py
│ ├── automl.py
│ ├── base_automl.py
│ ├── callbacks
│ │ ├── __init__.py
│ │ ├── callback_list.py
│ │ ├── callback.py
│ │ ├── early_stopping.py
│ │ ├── learner_time_constraint.py
│ │ ├── max_iters_constraint.py
│ │ ├── metric_logger.py
│ │ ├── terminate_on_nan.py
│ │ └── total_time_constraint.py
│ ├── ensemble.py
│ ├── exceptions.py
│ ├── fairness
│ │ ├── __init__.py
│ │ ├── metrics.py
│ │ ├── optimization.py
│ │ ├── plots.py
│ │ ├── report.py
│ │ └── utils.py
│ ├── model_framework.py
│ ├── preprocessing
│ │ ├── __init__.py
│ │ ├── datetime_transformer.py
│ │ ├── encoding_selector.py
│ │ ├── exclude_missing_target.py
│ │ ├── goldenfeatures_transformer.py
│ │ ├── kmeans_transformer.py
│ │ ├── label_binarizer.py
│ │ ├── label_encoder.py
│ │ ├── preprocessing_categorical.py
│ │ ├── preprocessing_missing.py
│ │ ├── preprocessing_utils.py
│ │ ├── preprocessing.py
│ │ ├── scale.py
│ │ └── text_transformer.py
│ ├── tuner
│ │ ├── __init__.py
│ │ ├── data_info.py
│ │ ├── hill_climbing.py
│ │ ├── mljar_tuner.py
│ │ ├── optuna
│ │ │ ├── __init__.py
│ │ │ ├── catboost.py
│ │ │ ├── extra_trees.py
│ │ │ ├── knn.py
│ │ │ ├── lightgbm.py
│ │ │ ├── nn.py
│ │ │ ├── random_forest.py
│ │ │ ├── tuner.py
│ │ │ └── xgboost.py
│ │ ├── preprocessing_tuner.py
│ │ ├── random_parameters.py
│ │ └── time_controller.py
│ ├── utils
│ │ ├── __init__.py
│ │ ├── additional_metrics.py
│ │ ├── additional_plots.py
│ │ ├── automl_plots.py
│ │ ├── common.py
│ │ ├── config.py
│ │ ├── constants.py
│ │ ├── data_validation.py
│ │ ├── importance.py
│ │ ├── jsonencoder.py
│ │ ├── leaderboard_plots.py
│ │ ├── learning_curves.py
│ │ ├── metric.py
│ │ ├── shap.py
│ │ ├── subsample.py
│ │ └── utils.py
│ └── validation
│ ├── __init__.py
│ ├── validation_step.py
│ ├── validator_base.py
│ ├── validator_custom.py
│ ├── validator_kfold.py
│ └── validator_split.py
└── tests
├── __init__.py
├── checks
│ ├── __init__.py
│ ├── check_automl_with_regression.py
│ ├── run_ml_tests.py
│ └── run_performance_tests.py
├── conftest.py
├── data
│ ├── 179.csv
│ ├── 24.csv
│ ├── 3.csv
│ ├── 31.csv
│ ├── 38.csv
│ ├── 44.csv
│ ├── 720.csv
│ ├── 737.csv
│ ├── acs_income_1k.csv
│ ├── adult_missing_values_missing_target_500rows.csv
│ ├── boston_housing.csv
│ ├── CrimeData
│ │ ├── cities.json
│ │ ├── crimedata.csv
│ │ └── README.md
│ ├── Drug
│ │ ├── Drug_Consumption.csv
│ │ └── README.md
│ ├── housing_regression_missing_values_missing_target.csv
│ ├── iris_classes_missing_values_missing_target.csv
│ ├── iris_missing_values_missing_target.csv
│ ├── LawSchool
│ │ ├── bar_pass_prediction.csv
│ │ └── README.md
│ ├── PortugeseBankMarketing
│ │ └── Data_FinalProject.csv
│ └── Titanic
│ ├── test_with_Survived.csv
│ └── train.csv
├── README.md
├── tests_algorithms
│ ├── __init__.py
│ ├── test_baseline.py
│ ├── test_catboost.py
│ ├── test_decision_tree.py
│ ├── test_extra_trees.py
│ ├── test_factory.py
│ ├── test_knn.py
│ ├── test_lightgbm.py
│ ├── test_linear.py
│ ├── test_nn.py
│ ├── test_random_forest.py
│ ├── test_registry.py
│ └── test_xgboost.py
├── tests_automl
│ ├── __init__.py
│ ├── test_adjust_validation.py
│ ├── test_automl_init.py
│ ├── test_automl_report.py
│ ├── test_automl_sample_weight.py
│ ├── test_automl_time_constraints.py
│ ├── test_automl.py
│ ├── test_data_types.py
│ ├── test_dir_change.py
│ ├── test_explain_levels.py
│ ├── test_golden_features.py
│ ├── test_handle_imbalance.py
│ ├── test_integration.py
│ ├── test_joblib_version.py
│ ├── test_models_needed_for_predict.py
│ ├── test_prediction_after_load.py
│ ├── test_repeated_validation.py
│ ├── test_restore.py
│ ├── test_stack_models_constraints.py
│ ├── test_targets.py
│ └── test_update_errors_report.py
├── tests_callbacks
│ ├── __init__.py
│ └── test_total_time_constraint.py
├── tests_ensemble
│ ├── __init__.py
│ └── test_save_load.py
├── tests_fairness
│ ├── __init__.py
│ ├── test_binary_classification.py
│ ├── test_multi_class_classification.py
│ └── test_regression.py
├── tests_preprocessing
│ ├── __init__.py
│ ├── disable_eda.py
│ ├── test_categorical_integers.py
│ ├── test_datetime_transformer.py
│ ├── test_encoding_selector.py
│ ├── test_exclude_missing.py
│ ├── test_goldenfeatures_transformer.py
│ ├── test_label_binarizer.py
│ ├── test_label_encoder.py
│ ├── test_preprocessing_missing.py
│ ├── test_preprocessing_utils.py
│ ├── test_preprocessing.py
│ ├── test_scale.py
│ └── test_text_transformer.py
├── tests_tuner
│ ├── __init__.py
│ ├── test_hill_climbing.py
│ ├── test_time_controller.py
│ └── test_tuner.py
├── tests_utils
│ ├── __init__.py
│ ├── test_compute_additional_metrics.py
│ ├── test_importance.py
│ ├── test_learning_curves.py
│ ├── test_metric.py
│ ├── test_shap.py
│ └── test_subsample.py
└── tests_validation
├── __init__.py
├── test_validator_kfold.py
└── test_validator_split.py
```
# Files
--------------------------------------------------------------------------------
/tests/data/Titanic/test_with_Survived.csv:
--------------------------------------------------------------------------------
```
1 | PassengerId,Survived,Pclass,Name,Sex,Age,SibSp,Parch,Ticket,Fare,Cabin,Embarked
2 | 892,0,3,"Kelly, Mr. James",male,34.5,0,0,330911,7.8292,,Q
3 | 893,1,3,"Wilkes, Mrs. James (Ellen Needs)",female,47,1,0,363272,7,,S
4 | 894,0,2,"Myles, Mr. Thomas Francis",male,62,0,0,240276,9.6875,,Q
5 | 895,0,3,"Wirz, Mr. Albert",male,27,0,0,315154,8.6625,,S
6 | 896,1,3,"Hirvonen, Mrs. Alexander (Helga E Lindqvist)",female,22,1,1,3101298,12.2875,,S
7 | 897,1,3,"Svensson, Mr. Johan Cervin",male,14,0,0,7538,9.225,,S
8 | 898,0,3,"Connolly, Miss. Kate",female,30,0,0,330972,7.6292,,Q
9 | 899,1,2,"Caldwell, Mr. Albert Francis",male,26,1,1,248738,29,,S
10 | 900,1,3,"Abrahim, Mrs. Joseph (Sophie Halaut Easu)",female,18,0,0,2657,7.2292,,C
11 | 901,0,3,"Davies, Mr. John Samuel",male,21,2,0,A/4 48871,24.15,,S
12 | 902,0,3,"Ilieff, Mr. Ylio",male,,0,0,349220,7.8958,,S
13 | 903,0,1,"Jones, Mr. Charles Cresson",male,46,0,0,694,26,,S
14 | 904,1,1,"Snyder, Mrs. John Pillsbury (Nelle Stevenson)",female,23,1,0,21228,82.2667,B45,S
15 | 905,0,2,"Howard, Mr. Benjamin",male,63,1,0,24065,26,,S
16 | 906,1,1,"Chaffee, Mrs. Herbert Fuller (Carrie Constance Toogood)",female,47,1,0,W.E.P. 5734,61.175,E31,S
17 | 907,1,2,"del Carlo, Mrs. Sebastiano (Argenia Genovesi)",female,24,1,0,SC/PARIS 2167,27.7208,,C
18 | 908,0,2,"Keane, Mr. Daniel",male,35,0,0,233734,12.35,,Q
19 | 909,0,3,"Assaf, Mr. Gerios",male,21,0,0,2692,7.225,,C
20 | 910,0,3,"Ilmakangas, Miss. Ida Livija",female,27,1,0,STON/O2. 3101270,7.925,,S
21 | 911,1,3,"Assaf Khalil, Mrs. Mariana (Miriam"")""",female,45,0,0,2696,7.225,,C
22 | 912,0,1,"Rothschild, Mr. Martin",male,55,1,0,PC 17603,59.4,,C
23 | 913,1,3,"Olsen, Master. Artur Karl",male,9,0,1,C 17368,3.1708,,S
24 | 914,1,1,"Flegenheim, Mrs. Alfred (Antoinette)",female,,0,0,PC 17598,31.6833,,S
25 | 915,1,1,"Williams, Mr. Richard Norris II",male,21,0,1,PC 17597,61.3792,,C
26 | 916,1,1,"Ryerson, Mrs. Arthur Larned (Emily Maria Borie)",female,48,1,3,PC 17608,262.375,B57 B59 B63 B66,C
27 | 917,0,3,"Robins, Mr. Alexander A",male,50,1,0,A/5. 3337,14.5,,S
28 | 918,1,1,"Ostby, Miss. Helene Ragnhild",female,22,0,1,113509,61.9792,B36,C
29 | 919,0,3,"Daher, Mr. Shedid",male,22.5,0,0,2698,7.225,,C
30 | 920,0,1,"Brady, Mr. John Bertram",male,41,0,0,113054,30.5,A21,S
31 | 921,0,3,"Samaan, Mr. Elias",male,,2,0,2662,21.6792,,C
32 | 922,0,2,"Louch, Mr. Charles Alexander",male,50,1,0,SC/AH 3085,26,,S
33 | 923,0,2,"Jefferys, Mr. Clifford Thomas",male,24,2,0,C.A. 31029,31.5,,S
34 | 924,1,3,"Dean, Mrs. Bertram (Eva Georgetta Light)",female,33,1,2,C.A. 2315,20.575,,S
35 | 925,0,3,"Johnston, Mrs. Andrew G (Elizabeth Lily"" Watson)""",female,,1,2,W./C. 6607,23.45,,S
36 | 926,1,1,"Mock, Mr. Philipp Edmund",male,30,1,0,13236,57.75,C78,C
37 | 927,0,3,"Katavelas, Mr. Vassilios (Catavelas Vassilios"")""",male,18.5,0,0,2682,7.2292,,C
38 | 928,1,3,"Roth, Miss. Sarah A",female,,0,0,342712,8.05,,S
39 | 929,0,3,"Cacic, Miss. Manda",female,21,0,0,315087,8.6625,,S
40 | 930,1,3,"Sap, Mr. Julius",male,25,0,0,345768,9.5,,S
41 | 931,1,3,"Hee, Mr. Ling",male,,0,0,1601,56.4958,,S
42 | 932,1,3,"Karun, Mr. Franz",male,39,0,1,349256,13.4167,,C
43 | 933,0,1,"Franklin, Mr. Thomas Parham",male,,0,0,113778,26.55,D34,S
44 | 934,0,3,"Goldsmith, Mr. Nathan",male,41,0,0,SOTON/O.Q. 3101263,7.85,,S
45 | 935,0,2,"Corbett, Mrs. Walter H (Irene Colvin)",female,30,0,0,237249,13,,S
46 | 936,1,1,"Kimball, Mrs. Edwin Nelson Jr (Gertrude Parsons)",female,45,1,0,11753,52.5542,D19,S
47 | 937,0,3,"Peltomaki, Mr. Nikolai Johannes",male,25,0,0,STON/O 2. 3101291,7.925,,S
48 | 938,1,1,"Chevre, Mr. Paul Romaine",male,45,0,0,PC 17594,29.7,A9,C
49 | 939,0,3,"Shaughnessy, Mr. Patrick",male,,0,0,370374,7.75,,Q
50 | 940,1,1,"Bucknell, Mrs. William Robert (Emma Eliza Ward)",female,60,0,0,11813,76.2917,D15,C
51 | 941,1,3,"Coutts, Mrs. William (Winnie Minnie"" Treanor)""",female,36,0,2,C.A. 37671,15.9,,S
52 | 942,0,1,"Smith, Mr. Lucien Philip",male,24,1,0,13695,60,C31,S
53 | 943,0,2,"Pulbaum, Mr. Franz",male,27,0,0,SC/PARIS 2168,15.0333,,C
54 | 944,1,2,"Hocking, Miss. Ellen Nellie""""",female,20,2,1,29105,23,,S
55 | 945,1,1,"Fortune, Miss. Ethel Flora",female,28,3,2,19950,263,C23 C25 C27,S
56 | 946,0,2,"Mangiavacchi, Mr. Serafino Emilio",male,,0,0,SC/A.3 2861,15.5792,,C
57 | 947,0,3,"Rice, Master. Albert",male,10,4,1,382652,29.125,,Q
58 | 948,0,3,"Cor, Mr. Bartol",male,35,0,0,349230,7.8958,,S
59 | 949,1,3,"Abelseth, Mr. Olaus Jorgensen",male,25,0,0,348122,7.65,F G63,S
60 | 950,0,3,"Davison, Mr. Thomas Henry",male,,1,0,386525,16.1,,S
61 | 951,1,1,"Chaudanson, Miss. Victorine",female,36,0,0,PC 17608,262.375,B61,C
62 | 952,0,3,"Dika, Mr. Mirko",male,17,0,0,349232,7.8958,,S
63 | 953,0,2,"McCrae, Mr. Arthur Gordon",male,32,0,0,237216,13.5,,S
64 | 954,0,3,"Bjorklund, Mr. Ernst Herbert",male,18,0,0,347090,7.75,,S
65 | 955,1,3,"Bradley, Miss. Bridget Delia",female,22,0,0,334914,7.725,,Q
66 | 956,1,1,"Ryerson, Master. John Borie",male,13,2,2,PC 17608,262.375,B57 B59 B63 B66,C
67 | 957,0,2,"Corey, Mrs. Percy C (Mary Phyllis Elizabeth Miller)",female,,0,0,F.C.C. 13534,21,,S
68 | 958,0,3,"Burns, Miss. Mary Delia",female,18,0,0,330963,7.8792,,Q
69 | 959,0,1,"Moore, Mr. Clarence Bloomfield",male,47,0,0,113796,42.4,,S
70 | 960,1,1,"Tucker, Mr. Gilbert Milligan Jr",male,31,0,0,2543,28.5375,C53,C
71 | 961,1,1,"Fortune, Mrs. Mark (Mary McDougald)",female,60,1,4,19950,263,C23 C25 C27,S
72 | 962,1,3,"Mulvihill, Miss. Bertha E",female,24,0,0,382653,7.75,,Q
73 | 963,0,3,"Minkoff, Mr. Lazar",male,21,0,0,349211,7.8958,,S
74 | 964,0,3,"Nieminen, Miss. Manta Josefina",female,29,0,0,3101297,7.925,,S
75 | 965,0,1,"Ovies y Rodriguez, Mr. Servando",male,28.5,0,0,PC 17562,27.7208,D43,C
76 | 966,1,1,"Geiger, Miss. Amalie",female,35,0,0,113503,211.5,C130,C
77 | 967,0,1,"Keeping, Mr. Edwin",male,32.5,0,0,113503,211.5,C132,C
78 | 968,0,3,"Miles, Mr. Frank",male,,0,0,359306,8.05,,S
79 | 969,1,1,"Cornell, Mrs. Robert Clifford (Malvina Helen Lamson)",female,55,2,0,11770,25.7,C101,S
80 | 970,0,2,"Aldworth, Mr. Charles Augustus",male,30,0,0,248744,13,,S
81 | 971,0,3,"Doyle, Miss. Elizabeth",female,24,0,0,368702,7.75,,Q
82 | 972,0,3,"Boulos, Master. Akar",male,6,1,1,2678,15.2458,,C
83 | 973,0,1,"Straus, Mr. Isidor",male,67,1,0,PC 17483,221.7792,C55 C57,S
84 | 974,0,1,"Case, Mr. Howard Brown",male,49,0,0,19924,26,,S
85 | 975,0,3,"Demetri, Mr. Marinko",male,,0,0,349238,7.8958,,S
86 | 976,0,2,"Lamb, Mr. John Joseph",male,,0,0,240261,10.7083,,Q
87 | 977,0,3,"Khalil, Mr. Betros",male,,1,0,2660,14.4542,,C
88 | 978,0,3,"Barry, Miss. Julia",female,27,0,0,330844,7.8792,,Q
89 | 979,1,3,"Badman, Miss. Emily Louisa",female,18,0,0,A/4 31416,8.05,,S
90 | 980,0,3,"O'Donoghue, Ms. Bridget",female,,0,0,364856,7.75,,Q
91 | 981,1,2,"Wells, Master. Ralph Lester",male,2,1,1,29103,23,,S
92 | 982,1,3,"Dyker, Mrs. Adolf Fredrik (Anna Elisabeth Judith Andersson)",female,22,1,0,347072,13.9,,S
93 | 983,0,3,"Pedersen, Mr. Olaf",male,,0,0,345498,7.775,,S
94 | 984,1,1,"Davidson, Mrs. Thornton (Orian Hays)",female,27,1,2,F.C. 12750,52,B71,S
95 | 985,0,3,"Guest, Mr. Robert",male,,0,0,376563,8.05,,S
96 | 986,0,1,"Birnbaum, Mr. Jakob",male,25,0,0,13905,26,,C
97 | 987,1,3,"Tenglin, Mr. Gunnar Isidor",male,25,0,0,350033,7.7958,,S
98 | 988,1,1,"Cavendish, Mrs. Tyrell William (Julia Florence Siegel)",female,76,1,0,19877,78.85,C46,S
99 | 989,0,3,"Makinen, Mr. Kalle Edvard",male,29,0,0,STON/O 2. 3101268,7.925,,S
100 | 990,0,3,"Braf, Miss. Elin Ester Maria",female,20,0,0,347471,7.8542,,S
101 | 991,0,3,"Nancarrow, Mr. William Henry",male,33,0,0,A./5. 3338,8.05,,S
102 | 992,1,1,"Stengel, Mrs. Charles Emil Henry (Annie May Morris)",female,43,1,0,11778,55.4417,C116,C
103 | 993,0,2,"Weisz, Mr. Leopold",male,27,1,0,228414,26,,S
104 | 994,0,3,"Foley, Mr. William",male,,0,0,365235,7.75,,Q
105 | 995,1,3,"Johansson Palmquist, Mr. Oskar Leander",male,26,0,0,347070,7.775,,S
106 | 996,1,3,"Thomas, Mrs. Alexander (Thamine Thelma"")""",female,16,1,1,2625,8.5167,,C
107 | 997,0,3,"Holthen, Mr. Johan Martin",male,28,0,0,C 4001,22.525,,S
108 | 998,1,3,"Buckley, Mr. Daniel",male,21,0,0,330920,7.8208,,Q
109 | 999,1,3,"Ryan, Mr. Edward",male,,0,0,383162,7.75,,Q
110 | 1000,0,3,"Willer, Mr. Aaron (Abi Weller"")""",male,,0,0,3410,8.7125,,S
111 | 1001,0,2,"Swane, Mr. George",male,18.5,0,0,248734,13,F,S
112 | 1002,0,2,"Stanton, Mr. Samuel Ward",male,41,0,0,237734,15.0458,,C
113 | 1003,1,3,"Shine, Miss. Ellen Natalia",female,,0,0,330968,7.7792,,Q
114 | 1004,0,1,"Evans, Miss. Edith Corse",female,36,0,0,PC 17531,31.6792,A29,C
115 | 1005,0,3,"Buckley, Miss. Katherine",female,18.5,0,0,329944,7.2833,,Q
116 | 1006,0,1,"Straus, Mrs. Isidor (Rosalie Ida Blun)",female,63,1,0,PC 17483,221.7792,C55 C57,S
117 | 1007,0,3,"Chronopoulos, Mr. Demetrios",male,18,1,0,2680,14.4542,,C
118 | 1008,0,3,"Thomas, Mr. John",male,,0,0,2681,6.4375,,C
119 | 1009,1,3,"Sandstrom, Miss. Beatrice Irene",female,1,1,1,PP 9549,16.7,G6,S
120 | 1010,0,1,"Beattie, Mr. Thomson",male,36,0,0,13050,75.2417,C6,C
121 | 1011,0,2,"Chapman, Mrs. John Henry (Sara Elizabeth Lawry)",female,29,1,0,SC/AH 29037,26,,S
122 | 1012,1,2,"Watt, Miss. Bertha J",female,12,0,0,C.A. 33595,15.75,,S
123 | 1013,0,3,"Kiernan, Mr. John",male,,1,0,367227,7.75,,Q
124 | 1014,1,1,"Schabert, Mrs. Paul (Emma Mock)",female,35,1,0,13236,57.75,C28,C
125 | 1015,0,3,"Carver, Mr. Alfred John",male,28,0,0,392095,7.25,,S
126 | 1016,1,3,"Kennedy, Mr. John",male,,0,0,368783,7.75,,Q
127 | 1017,1,3,"Cribb, Miss. Laura Alice",female,17,0,1,371362,16.1,,S
128 | 1018,0,3,"Brobeck, Mr. Karl Rudolf",male,22,0,0,350045,7.7958,,S
129 | 1019,1,3,"McCoy, Miss. Alicia",female,,2,0,367226,23.25,,Q
130 | 1020,0,2,"Bowenur, Mr. Solomon",male,42,0,0,211535,13,,S
131 | 1021,0,3,"Petersen, Mr. Marius",male,24,0,0,342441,8.05,,S
132 | 1022,0,3,"Spinner, Mr. Henry John",male,32,0,0,STON/OQ. 369943,8.05,,S
133 | 1023,1,1,"Gracie, Col. Archibald IV",male,53,0,0,113780,28.5,C51,C
134 | 1024,0,3,"Lefebre, Mrs. Frank (Frances)",female,,0,4,4133,25.4667,,S
135 | 1025,0,3,"Thomas, Mr. Charles P",male,,1,0,2621,6.4375,,C
136 | 1026,0,3,"Dintcheff, Mr. Valtcho",male,43,0,0,349226,7.8958,,S
137 | 1027,0,3,"Carlsson, Mr. Carl Robert",male,24,0,0,350409,7.8542,,S
138 | 1028,0,3,"Zakarian, Mr. Mapriededer",male,26.5,0,0,2656,7.225,,C
139 | 1029,0,2,"Schmidt, Mr. August",male,26,0,0,248659,13,,S
140 | 1030,1,3,"Drapkin, Miss. Jennie",female,23,0,0,SOTON/OQ 392083,8.05,,S
141 | 1031,0,3,"Goodwin, Mr. Charles Frederick",male,40,1,6,CA 2144,46.9,,S
142 | 1032,0,3,"Goodwin, Miss. Jessie Allis",female,10,5,2,CA 2144,46.9,,S
143 | 1033,1,1,"Daniels, Miss. Sarah",female,33,0,0,113781,151.55,,S
144 | 1034,0,1,"Ryerson, Mr. Arthur Larned",male,61,1,3,PC 17608,262.375,B57 B59 B63 B66,C
145 | 1035,0,2,"Beauchamp, Mr. Henry James",male,28,0,0,244358,26,,S
146 | 1036,0,1,"Lindeberg-Lind, Mr. Erik Gustaf (Mr Edward Lingrey"")""",male,42,0,0,17475,26.55,,S
147 | 1037,0,3,"Vander Planke, Mr. Julius",male,31,3,0,345763,18,,S
148 | 1038,0,1,"Hilliard, Mr. Herbert Henry",male,,0,0,17463,51.8625,E46,S
149 | 1039,0,3,"Davies, Mr. Evan",male,22,0,0,SC/A4 23568,8.05,,S
150 | 1040,0,1,"Crafton, Mr. John Bertram",male,,0,0,113791,26.55,,S
151 | 1041,0,2,"Lahtinen, Rev. William",male,30,1,1,250651,26,,S
152 | 1042,1,1,"Earnshaw, Mrs. Boulton (Olive Potter)",female,23,0,1,11767,83.1583,C54,C
153 | 1043,0,3,"Matinoff, Mr. Nicola",male,,0,0,349255,7.8958,,C
154 | 1044,0,3,"Storey, Mr. Thomas",male,60.5,0,0,3701,,,S
155 | 1045,0,3,"Klasen, Mrs. (Hulda Kristina Eugenia Lofqvist)",female,36,0,2,350405,12.1833,,S
156 | 1046,0,3,"Asplund, Master. Filip Oscar",male,13,4,2,347077,31.3875,,S
157 | 1047,1,3,"Duquemin, Mr. Joseph",male,24,0,0,S.O./P.P. 752,7.55,,S
158 | 1048,1,1,"Bird, Miss. Ellen",female,29,0,0,PC 17483,221.7792,C97,S
159 | 1049,1,3,"Lundin, Miss. Olga Elida",female,23,0,0,347469,7.8542,,S
160 | 1050,0,1,"Borebank, Mr. John James",male,42,0,0,110489,26.55,D22,S
161 | 1051,0,3,"Peacock, Mrs. Benjamin (Edith Nile)",female,26,0,2,SOTON/O.Q. 3101315,13.775,,S
162 | 1052,1,3,"Smyth, Miss. Julia",female,,0,0,335432,7.7333,,Q
163 | 1053,1,3,"Touma, Master. Georges Youssef",male,7,1,1,2650,15.2458,,C
164 | 1054,1,2,"Wright, Miss. Marion",female,26,0,0,220844,13.5,,S
165 | 1055,0,3,"Pearce, Mr. Ernest",male,,0,0,343271,7,,S
166 | 1056,0,2,"Peruschitz, Rev. Joseph Maria",male,41,0,0,237393,13,,S
167 | 1057,1,3,"Kink-Heilmann, Mrs. Anton (Luise Heilmann)",female,26,1,1,315153,22.025,,S
168 | 1058,0,1,"Brandeis, Mr. Emil",male,48,0,0,PC 17591,50.4958,B10,C
169 | 1059,0,3,"Ford, Mr. Edward Watson",male,18,2,2,W./C. 6608,34.375,,S
170 | 1060,1,1,"Cassebeer, Mrs. Henry Arthur Jr (Eleanor Genevieve Fosdick)",female,,0,0,17770,27.7208,,C
171 | 1061,1,3,"Hellstrom, Miss. Hilda Maria",female,22,0,0,7548,8.9625,,S
172 | 1062,0,3,"Lithman, Mr. Simon",male,,0,0,S.O./P.P. 251,7.55,,S
173 | 1063,0,3,"Zakarian, Mr. Ortin",male,27,0,0,2670,7.225,,C
174 | 1064,0,3,"Dyker, Mr. Adolf Fredrik",male,23,1,0,347072,13.9,,S
175 | 1065,0,3,"Torfa, Mr. Assad",male,,0,0,2673,7.2292,,C
176 | 1066,0,3,"Asplund, Mr. Carl Oscar Vilhelm Gustafsson",male,40,1,5,347077,31.3875,,S
177 | 1067,1,2,"Brown, Miss. Edith Eileen",female,15,0,2,29750,39,,S
178 | 1068,1,2,"Sincock, Miss. Maude",female,20,0,0,C.A. 33112,36.75,,S
179 | 1069,1,1,"Stengel, Mr. Charles Emil Henry",male,54,1,0,11778,55.4417,C116,C
180 | 1070,1,2,"Becker, Mrs. Allen Oliver (Nellie E Baumgardner)",female,36,0,3,230136,39,F4,S
181 | 1071,1,1,"Compton, Mrs. Alexander Taylor (Mary Eliza Ingersoll)",female,64,0,2,PC 17756,83.1583,E45,C
182 | 1072,0,2,"McCrie, Mr. James Matthew",male,30,0,0,233478,13,,S
183 | 1073,0,1,"Compton, Mr. Alexander Taylor Jr",male,37,1,1,PC 17756,83.1583,E52,C
184 | 1074,1,1,"Marvin, Mrs. Daniel Warner (Mary Graham Carmichael Farquarson)",female,18,1,0,113773,53.1,D30,S
185 | 1075,0,3,"Lane, Mr. Patrick",male,,0,0,7935,7.75,,Q
186 | 1076,1,1,"Douglas, Mrs. Frederick Charles (Mary Helene Baxter)",female,27,1,1,PC 17558,247.5208,B58 B60,C
187 | 1077,0,2,"Maybery, Mr. Frank Hubert",male,40,0,0,239059,16,,S
188 | 1078,1,2,"Phillips, Miss. Alice Frances Louisa",female,21,0,1,S.O./P.P. 2,21,,S
189 | 1079,0,3,"Davies, Mr. Joseph",male,17,2,0,A/4 48873,8.05,,S
190 | 1080,0,3,"Sage, Miss. Ada",female,,8,2,CA. 2343,69.55,,S
191 | 1081,0,2,"Veal, Mr. James",male,40,0,0,28221,13,,S
192 | 1082,0,2,"Angle, Mr. William A",male,34,1,0,226875,26,,S
193 | 1083,1,1,"Salomon, Mr. Abraham L",male,,0,0,111163,26,,S
194 | 1084,0,3,"van Billiard, Master. Walter John",male,11.5,1,1,A/5. 851,14.5,,S
195 | 1085,0,2,"Lingane, Mr. John",male,61,0,0,235509,12.35,,Q
196 | 1086,1,2,"Drew, Master. Marshall Brines",male,8,0,2,28220,32.5,,S
197 | 1087,0,3,"Karlsson, Mr. Julius Konrad Eugen",male,33,0,0,347465,7.8542,,S
198 | 1088,1,1,"Spedden, Master. Robert Douglas",male,6,0,2,16966,134.5,E34,C
199 | 1089,1,3,"Nilsson, Miss. Berta Olivia",female,18,0,0,347066,7.775,,S
200 | 1090,0,2,"Baimbrigge, Mr. Charles Robert",male,23,0,0,C.A. 31030,10.5,,S
201 | 1091,0,3,"Rasmussen, Mrs. (Lena Jacobsen Solvang)",female,,0,0,65305,8.1125,,S
202 | 1092,1,3,"Murphy, Miss. Nora",female,,0,0,36568,15.5,,Q
203 | 1093,0,3,"Danbom, Master. Gilbert Sigvard Emanuel",male,0.33,0,2,347080,14.4,,S
204 | 1094,0,1,"Astor, Col. John Jacob",male,47,1,0,PC 17757,227.525,C62 C64,C
205 | 1095,1,2,"Quick, Miss. Winifred Vera",female,8,1,1,26360,26,,S
206 | 1096,0,2,"Andrew, Mr. Frank Thomas",male,25,0,0,C.A. 34050,10.5,,S
207 | 1097,1,1,"Omont, Mr. Alfred Fernand",male,,0,0,F.C. 12998,25.7417,,C
208 | 1098,0,3,"McGowan, Miss. Katherine",female,35,0,0,9232,7.75,,Q
209 | 1099,1,2,"Collett, Mr. Sidney C Stuart",male,24,0,0,28034,10.5,,S
210 | 1100,1,1,"Rosenbaum, Miss. Edith Louise",female,33,0,0,PC 17613,27.7208,A11,C
211 | 1101,0,3,"Delalic, Mr. Redjo",male,25,0,0,349250,7.8958,,S
212 | 1102,0,3,"Andersen, Mr. Albert Karvin",male,32,0,0,C 4001,22.525,,S
213 | 1103,1,3,"Finoli, Mr. Luigi",male,,0,0,SOTON/O.Q. 3101308,7.05,,S
214 | 1104,0,2,"Deacon, Mr. Percy William",male,17,0,0,S.O.C. 14879,73.5,,S
215 | 1105,0,2,"Howard, Mrs. Benjamin (Ellen Truelove Arman)",female,60,1,0,24065,26,,S
216 | 1106,0,3,"Andersson, Miss. Ida Augusta Margareta",female,38,4,2,347091,7.775,,S
217 | 1107,0,1,"Head, Mr. Christopher",male,42,0,0,113038,42.5,B11,S
218 | 1108,0,3,"Mahon, Miss. Bridget Delia",female,,0,0,330924,7.8792,,Q
219 | 1109,0,1,"Wick, Mr. George Dennick",male,57,1,1,36928,164.8667,,S
220 | 1110,1,1,"Widener, Mrs. George Dunton (Eleanor Elkins)",female,50,1,1,113503,211.5,C80,C
221 | 1111,0,3,"Thomson, Mr. Alexander Morrison",male,,0,0,32302,8.05,,S
222 | 1112,1,2,"Duran y More, Miss. Florentina",female,30,1,0,SC/PARIS 2148,13.8583,,C
223 | 1113,0,3,"Reynolds, Mr. Harold J",male,21,0,0,342684,8.05,,S
224 | 1114,1,2,"Cook, Mrs. (Selena Rogers)",female,22,0,0,W./C. 14266,10.5,F33,S
225 | 1115,1,3,"Karlsson, Mr. Einar Gervasius",male,21,0,0,350053,7.7958,,S
226 | 1116,1,1,"Candee, Mrs. Edward (Helen Churchill Hungerford)",female,53,0,0,PC 17606,27.4458,,C
227 | 1117,1,3,"Moubarek, Mrs. George (Omine Amenia"" Alexander)""",female,,0,2,2661,15.2458,,C
228 | 1118,1,3,"Asplund, Mr. Johan Charles",male,23,0,0,350054,7.7958,,S
229 | 1119,0,3,"McNeill, Miss. Bridget",female,,0,0,370368,7.75,,Q
230 | 1120,0,3,"Everett, Mr. Thomas James",male,40.5,0,0,C.A. 6212,15.1,,S
231 | 1121,0,2,"Hocking, Mr. Samuel James Metcalfe",male,36,0,0,242963,13,,S
232 | 1122,0,2,"Sweet, Mr. George Frederick",male,14,0,0,220845,65,,S
233 | 1123,1,1,"Willard, Miss. Constance",female,21,0,0,113795,26.55,,S
234 | 1124,0,3,"Wiklund, Mr. Karl Johan",male,21,1,0,3101266,6.4958,,S
235 | 1125,0,3,"Linehan, Mr. Michael",male,,0,0,330971,7.8792,,Q
236 | 1126,0,1,"Cumings, Mr. John Bradley",male,39,1,0,PC 17599,71.2833,C85,C
237 | 1127,0,3,"Vendel, Mr. Olof Edvin",male,20,0,0,350416,7.8542,,S
238 | 1128,0,1,"Warren, Mr. Frank Manley",male,64,1,0,110813,75.25,D37,C
239 | 1129,0,3,"Baccos, Mr. Raffull",male,20,0,0,2679,7.225,,C
240 | 1130,0,2,"Hiltunen, Miss. Marta",female,18,1,1,250650,13,,S
241 | 1131,1,1,"Douglas, Mrs. Walter Donald (Mahala Dutton)",female,48,1,0,PC 17761,106.425,C86,C
242 | 1132,1,1,"Lindstrom, Mrs. Carl Johan (Sigrid Posse)",female,55,0,0,112377,27.7208,,C
243 | 1133,1,2,"Christy, Mrs. (Alice Frances)",female,45,0,2,237789,30,,S
244 | 1134,1,1,"Spedden, Mr. Frederic Oakley",male,45,1,1,16966,134.5,E34,C
245 | 1135,1,3,"Hyman, Mr. Abraham",male,,0,0,3470,7.8875,,S
246 | 1136,0,3,"Johnston, Master. William Arthur Willie""""",male,,1,2,W./C. 6607,23.45,,S
247 | 1137,0,1,"Kenyon, Mr. Frederick R",male,41,1,0,17464,51.8625,D21,S
248 | 1138,0,2,"Karnes, Mrs. J Frank (Claire Bennett)",female,22,0,0,F.C.C. 13534,21,,S
249 | 1139,0,2,"Drew, Mr. James Vivian",male,42,1,1,28220,32.5,,S
250 | 1140,1,2,"Hold, Mrs. Stephen (Annie Margaret Hill)",female,29,1,0,26707,26,,S
251 | 1141,0,3,"Khalil, Mrs. Betros (Zahie Maria"" Elias)""",female,,1,0,2660,14.4542,,C
252 | 1142,1,2,"West, Miss. Barbara J",female,0.92,1,2,C.A. 34651,27.75,,S
253 | 1143,1,3,"Abrahamsson, Mr. Abraham August Johannes",male,20,0,0,SOTON/O2 3101284,7.925,,S
254 | 1144,0,1,"Clark, Mr. Walter Miller",male,27,1,0,13508,136.7792,C89,C
255 | 1145,0,3,"Salander, Mr. Karl Johan",male,24,0,0,7266,9.325,,S
256 | 1146,0,3,"Wenzel, Mr. Linhart",male,32.5,0,0,345775,9.5,,S
257 | 1147,0,3,"MacKay, Mr. George William",male,,0,0,C.A. 42795,7.55,,S
258 | 1148,0,3,"Mahon, Mr. John",male,,0,0,AQ/4 3130,7.75,,Q
259 | 1149,0,3,"Niklasson, Mr. Samuel",male,28,0,0,363611,8.05,,S
260 | 1150,1,2,"Bentham, Miss. Lilian W",female,19,0,0,28404,13,,S
261 | 1151,1,3,"Midtsjo, Mr. Karl Albert",male,21,0,0,345501,7.775,,S
262 | 1152,1,3,"de Messemaeker, Mr. Guillaume Joseph",male,36.5,1,0,345572,17.4,,S
263 | 1153,0,3,"Nilsson, Mr. August Ferdinand",male,21,0,0,350410,7.8542,,S
264 | 1154,1,2,"Wells, Mrs. Arthur Henry (Addie"" Dart Trevaskis)""",female,29,0,2,29103,23,,S
265 | 1155,0,3,"Klasen, Miss. Gertrud Emilia",female,1,1,1,350405,12.1833,,S
266 | 1156,1,2,"Portaluppi, Mr. Emilio Ilario Giuseppe",male,30,0,0,C.A. 34644,12.7375,,C
267 | 1157,0,3,"Lyntakoff, Mr. Stanko",male,,0,0,349235,7.8958,,S
268 | 1158,0,1,"Chisholm, Mr. Roderick Robert Crispin",male,,0,0,112051,0,,S
269 | 1159,0,3,"Warren, Mr. Charles William",male,,0,0,C.A. 49867,7.55,,S
270 | 1160,1,3,"Howard, Miss. May Elizabeth",female,,0,0,A. 2. 39186,8.05,,S
271 | 1161,0,3,"Pokrnic, Mr. Mate",male,17,0,0,315095,8.6625,,S
272 | 1162,0,1,"McCaffry, Mr. Thomas Francis",male,46,0,0,13050,75.2417,C6,C
273 | 1163,0,3,"Fox, Mr. Patrick",male,,0,0,368573,7.75,,Q
274 | 1164,1,1,"Clark, Mrs. Walter Miller (Virginia McDowell)",female,26,1,0,13508,136.7792,C89,C
275 | 1165,0,3,"Lennon, Miss. Mary",female,,1,0,370371,15.5,,Q
276 | 1166,0,3,"Saade, Mr. Jean Nassr",male,,0,0,2676,7.225,,C
277 | 1167,1,2,"Bryhl, Miss. Dagmar Jenny Ingeborg ",female,20,1,0,236853,26,,S
278 | 1168,0,2,"Parker, Mr. Clifford Richard",male,28,0,0,SC 14888,10.5,,S
279 | 1169,0,2,"Faunthorpe, Mr. Harry",male,40,1,0,2926,26,,S
280 | 1170,0,2,"Ware, Mr. John James",male,30,1,0,CA 31352,21,,S
281 | 1171,1,2,"Oxenham, Mr. Percy Thomas",male,22,0,0,W./C. 14260,10.5,,S
282 | 1172,0,3,"Oreskovic, Miss. Jelka",female,23,0,0,315085,8.6625,,S
283 | 1173,0,3,"Peacock, Master. Alfred Edward",male,0.75,1,1,SOTON/O.Q. 3101315,13.775,,S
284 | 1174,0,3,"Fleming, Miss. Honora",female,,0,0,364859,7.75,,Q
285 | 1175,1,3,"Touma, Miss. Maria Youssef",female,9,1,1,2650,15.2458,,C
286 | 1176,0,3,"Rosblom, Miss. Salli Helena",female,2,1,1,370129,20.2125,,S
287 | 1177,0,3,"Dennis, Mr. William",male,36,0,0,A/5 21175,7.25,,S
288 | 1178,0,3,"Franklin, Mr. Charles (Charles Fardon)",male,,0,0,SOTON/O.Q. 3101314,7.25,,S
289 | 1179,1,1,"Snyder, Mr. John Pillsbury",male,24,1,0,21228,82.2667,B45,S
290 | 1180,0,3,"Mardirosian, Mr. Sarkis",male,,0,0,2655,7.2292,F E46,C
291 | 1181,0,3,"Ford, Mr. Arthur",male,,0,0,A/5 1478,8.05,,S
292 | 1182,1,1,"Rheims, Mr. George Alexander Lucien",male,,0,0,PC 17607,39.6,,S
293 | 1183,1,3,"Daly, Miss. Margaret Marcella Maggie""""",female,30,0,0,382650,6.95,,Q
294 | 1184,0,3,"Nasr, Mr. Mustafa",male,,0,0,2652,7.2292,,C
295 | 1185,1,1,"Dodge, Dr. Washington",male,53,1,1,33638,81.8583,A34,S
296 | 1186,0,3,"Wittevrongel, Mr. Camille",male,36,0,0,345771,9.5,,S
297 | 1187,0,3,"Angheloff, Mr. Minko",male,26,0,0,349202,7.8958,,S
298 | 1188,1,2,"Laroche, Miss. Louise",female,1,1,2,SC/Paris 2123,41.5792,,C
299 | 1189,0,3,"Samaan, Mr. Hanna",male,,2,0,2662,21.6792,,C
300 | 1190,0,1,"Loring, Mr. Joseph Holland",male,30,0,0,113801,45.5,,S
301 | 1191,0,3,"Johansson, Mr. Nils",male,29,0,0,347467,7.8542,,S
302 | 1192,1,3,"Olsson, Mr. Oscar Wilhelm",male,32,0,0,347079,7.775,,S
303 | 1193,0,2,"Malachard, Mr. Noel",male,,0,0,237735,15.0458,D,C
304 | 1194,0,2,"Phillips, Mr. Escott Robert",male,43,0,1,S.O./P.P. 2,21,,S
305 | 1195,0,3,"Pokrnic, Mr. Tome",male,24,0,0,315092,8.6625,,S
306 | 1196,1,3,"McCarthy, Miss. Catherine Katie""""",female,,0,0,383123,7.75,,Q
307 | 1197,1,1,"Crosby, Mrs. Edward Gifford (Catherine Elizabeth Halstead)",female,64,1,1,112901,26.55,B26,S
308 | 1198,0,1,"Allison, Mr. Hudson Joshua Creighton",male,30,1,2,113781,151.55,C22 C26,S
309 | 1199,1,3,"Aks, Master. Philip Frank",male,0.83,0,1,392091,9.35,,S
310 | 1200,0,1,"Hays, Mr. Charles Melville",male,55,1,1,12749,93.5,B69,S
311 | 1201,1,3,"Hansen, Mrs. Claus Peter (Jennie L Howard)",female,45,1,0,350026,14.1083,,S
312 | 1202,0,3,"Cacic, Mr. Jego Grga",male,18,0,0,315091,8.6625,,S
313 | 1203,1,3,"Vartanian, Mr. David",male,22,0,0,2658,7.225,,C
314 | 1204,0,3,"Sadowitz, Mr. Harry",male,,0,0,LP 1588,7.575,,S
315 | 1205,0,3,"Carr, Miss. Jeannie",female,37,0,0,368364,7.75,,Q
316 | 1206,1,1,"White, Mrs. John Stuart (Ella Holmes)",female,55,0,0,PC 17760,135.6333,C32,C
317 | 1207,0,3,"Hagardon, Miss. Kate",female,17,0,0,AQ/3. 30631,7.7333,,Q
318 | 1208,0,1,"Spencer, Mr. William Augustus",male,57,1,0,PC 17569,146.5208,B78,C
319 | 1209,0,2,"Rogers, Mr. Reginald Harry",male,19,0,0,28004,10.5,,S
320 | 1210,0,3,"Jonsson, Mr. Nils Hilding",male,27,0,0,350408,7.8542,,S
321 | 1211,0,2,"Jefferys, Mr. Ernest Wilfred",male,22,2,0,C.A. 31029,31.5,,S
322 | 1212,0,3,"Andersson, Mr. Johan Samuel",male,26,0,0,347075,7.775,,S
323 | 1213,1,3,"Krekorian, Mr. Neshan",male,25,0,0,2654,7.2292,F E57,C
324 | 1214,0,2,"Nesson, Mr. Israel",male,26,0,0,244368,13,F2,S
325 | 1215,0,1,"Rowe, Mr. Alfred G",male,33,0,0,113790,26.55,,S
326 | 1216,1,1,"Kreuchen, Miss. Emilie",female,39,0,0,24160,211.3375,,S
327 | 1217,0,3,"Assam, Mr. Ali",male,23,0,0,SOTON/O.Q. 3101309,7.05,,S
328 | 1218,1,2,"Becker, Miss. Ruth Elizabeth",female,12,2,1,230136,39,F4,S
329 | 1219,0,1,"Rosenshine, Mr. George (Mr George Thorne"")""",male,46,0,0,PC 17585,79.2,,C
330 | 1220,0,2,"Clarke, Mr. Charles Valentine",male,29,1,0,2003,26,,S
331 | 1221,0,2,"Enander, Mr. Ingvar",male,21,0,0,236854,13,,S
332 | 1222,1,2,"Davies, Mrs. John Morgan (Elizabeth Agnes Mary White) ",female,48,0,2,C.A. 33112,36.75,,S
333 | 1223,0,1,"Dulles, Mr. William Crothers",male,39,0,0,PC 17580,29.7,A18,C
334 | 1224,0,3,"Thomas, Mr. Tannous",male,,0,0,2684,7.225,,C
335 | 1225,1,3,"Nakid, Mrs. Said (Waika Mary"" Mowad)""",female,19,1,1,2653,15.7417,,C
336 | 1226,0,3,"Cor, Mr. Ivan",male,27,0,0,349229,7.8958,,S
337 | 1227,0,1,"Maguire, Mr. John Edward",male,30,0,0,110469,26,C106,S
338 | 1228,0,2,"de Brito, Mr. Jose Joaquim",male,32,0,0,244360,13,,S
339 | 1229,0,3,"Elias, Mr. Joseph",male,39,0,2,2675,7.2292,,C
340 | 1230,0,2,"Denbury, Mr. Herbert",male,25,0,0,C.A. 31029,31.5,,S
341 | 1231,0,3,"Betros, Master. Seman",male,,0,0,2622,7.2292,,C
342 | 1232,0,2,"Fillbrook, Mr. Joseph Charles",male,18,0,0,C.A. 15185,10.5,,S
343 | 1233,1,3,"Lundstrom, Mr. Thure Edvin",male,32,0,0,350403,7.5792,,S
344 | 1234,0,3,"Sage, Mr. John George",male,,1,9,CA. 2343,69.55,,S
345 | 1235,1,1,"Cardeza, Mrs. James Warburton Martinez (Charlotte Wardle Drake)",female,58,0,1,PC 17755,512.3292,B51 B53 B55,C
346 | 1236,0,3,"van Billiard, Master. James William",male,,1,1,A/5. 851,14.5,,S
347 | 1237,1,3,"Abelseth, Miss. Karen Marie",female,16,0,0,348125,7.65,,S
348 | 1238,0,2,"Botsford, Mr. William Hull",male,26,0,0,237670,13,,S
349 | 1239,1,3,"Whabee, Mrs. George Joseph (Shawneene Abi-Saab)",female,38,0,0,2688,7.2292,,C
350 | 1240,0,2,"Giles, Mr. Ralph",male,24,0,0,248726,13.5,,S
351 | 1241,1,2,"Walcroft, Miss. Nellie",female,31,0,0,F.C.C. 13528,21,,S
352 | 1242,1,1,"Greenfield, Mrs. Leo David (Blanche Strouse)",female,45,0,1,PC 17759,63.3583,D10 D12,C
353 | 1243,0,2,"Stokes, Mr. Philip Joseph",male,25,0,0,F.C.C. 13540,10.5,,S
354 | 1244,0,2,"Dibden, Mr. William",male,18,0,0,S.O.C. 14879,73.5,,S
355 | 1245,0,2,"Herman, Mr. Samuel",male,49,1,2,220845,65,,S
356 | 1246,1,3,"Dean, Miss. Elizabeth Gladys Millvina""""",female,0.17,1,2,C.A. 2315,20.575,,S
357 | 1247,0,1,"Julian, Mr. Henry Forbes",male,50,0,0,113044,26,E60,S
358 | 1248,1,1,"Brown, Mrs. John Murray (Caroline Lane Lamson)",female,59,2,0,11769,51.4792,C101,S
359 | 1249,0,3,"Lockyer, Mr. Edward",male,,0,0,1222,7.8792,,S
360 | 1250,1,3,"O'Keefe, Mr. Patrick",male,,0,0,368402,7.75,,Q
361 | 1251,0,3,"Lindell, Mrs. Edvard Bengtsson (Elin Gerda Persson)",female,30,1,0,349910,15.55,,S
362 | 1252,0,3,"Sage, Master. William Henry",male,14.5,8,2,CA. 2343,69.55,,S
363 | 1253,1,2,"Mallet, Mrs. Albert (Antoinette Magnin)",female,24,1,1,S.C./PARIS 2079,37.0042,,C
364 | 1254,1,2,"Ware, Mrs. John James (Florence Louise Long)",female,31,0,0,CA 31352,21,,S
365 | 1255,0,3,"Strilic, Mr. Ivan",male,27,0,0,315083,8.6625,,S
366 | 1256,1,1,"Harder, Mrs. George Achilles (Dorothy Annan)",female,25,1,0,11765,55.4417,E50,C
367 | 1257,0,3,"Sage, Mrs. John (Annie Bullen)",female,,1,9,CA. 2343,69.55,,S
368 | 1258,0,3,"Caram, Mr. Joseph",male,,1,0,2689,14.4583,,C
369 | 1259,0,3,"Riihivouri, Miss. Susanna Juhantytar Sanni""""",female,22,0,0,3101295,39.6875,,S
370 | 1260,1,1,"Gibson, Mrs. Leonard (Pauline C Boeson)",female,45,0,1,112378,59.4,,C
371 | 1261,1,2,"Pallas y Castello, Mr. Emilio",male,29,0,0,SC/PARIS 2147,13.8583,,C
372 | 1262,0,2,"Giles, Mr. Edgar",male,21,1,0,28133,11.5,,S
373 | 1263,1,1,"Wilson, Miss. Helen Alice",female,31,0,0,16966,134.5,E39 E41,C
374 | 1264,1,1,"Ismay, Mr. Joseph Bruce",male,49,0,0,112058,0,B52 B54 B56,S
375 | 1265,0,2,"Harbeck, Mr. William H",male,44,0,0,248746,13,,S
376 | 1266,1,1,"Dodge, Mrs. Washington (Ruth Vidaver)",female,54,1,1,33638,81.8583,A34,S
377 | 1267,1,1,"Bowen, Miss. Grace Scott",female,45,0,0,PC 17608,262.375,,C
378 | 1268,0,3,"Kink, Miss. Maria",female,22,2,0,315152,8.6625,,S
379 | 1269,0,2,"Cotterill, Mr. Henry Harry""""",male,21,0,0,29107,11.5,,S
380 | 1270,0,1,"Hipkins, Mr. William Edward",male,55,0,0,680,50,C39,S
381 | 1271,0,3,"Asplund, Master. Carl Edgar",male,5,4,2,347077,31.3875,,S
382 | 1272,0,3,"O'Connor, Mr. Patrick",male,,0,0,366713,7.75,,Q
383 | 1273,0,3,"Foley, Mr. Joseph",male,26,0,0,330910,7.8792,,Q
384 | 1274,0,3,"Risien, Mrs. Samuel (Emma)",female,,0,0,364498,14.5,,S
385 | 1275,0,3,"McNamee, Mrs. Neal (Eileen O'Leary)",female,19,1,0,376566,16.1,,S
386 | 1276,0,2,"Wheeler, Mr. Edwin Frederick""""",male,,0,0,SC/PARIS 2159,12.875,,S
387 | 1277,1,2,"Herman, Miss. Kate",female,24,1,2,220845,65,,S
388 | 1278,0,3,"Aronsson, Mr. Ernst Axel Algot",male,24,0,0,349911,7.775,,S
389 | 1279,0,2,"Ashby, Mr. John",male,57,0,0,244346,13,,S
390 | 1280,0,3,"Canavan, Mr. Patrick",male,21,0,0,364858,7.75,,Q
391 | 1281,0,3,"Palsson, Master. Paul Folke",male,6,3,1,349909,21.075,,S
392 | 1282,0,1,"Payne, Mr. Vivian Ponsonby",male,23,0,0,12749,93.5,B24,S
393 | 1283,1,1,"Lines, Mrs. Ernest H (Elizabeth Lindsey James)",female,51,0,1,PC 17592,39.4,D28,S
394 | 1284,0,3,"Abbott, Master. Eugene Joseph",male,13,0,2,C.A. 2673,20.25,,S
395 | 1285,0,2,"Gilbert, Mr. William",male,47,0,0,C.A. 30769,10.5,,S
396 | 1286,1,3,"Kink-Heilmann, Mr. Anton",male,29,3,1,315153,22.025,,S
397 | 1287,1,1,"Smith, Mrs. Lucien Philip (Mary Eloise Hughes)",female,18,1,0,13695,60,C31,S
398 | 1288,0,3,"Colbert, Mr. Patrick",male,24,0,0,371109,7.25,,Q
399 | 1289,1,1,"Frolicher-Stehli, Mrs. Maxmillian (Margaretha Emerentia Stehli)",female,48,1,1,13567,79.2,B41,C
400 | 1290,0,3,"Larsson-Rondberg, Mr. Edvard A",male,22,0,0,347065,7.775,,S
401 | 1291,0,3,"Conlon, Mr. Thomas Henry",male,31,0,0,21332,7.7333,,Q
402 | 1292,1,1,"Bonnell, Miss. Caroline",female,30,0,0,36928,164.8667,C7,S
403 | 1293,0,2,"Gale, Mr. Harry",male,38,1,0,28664,21,,S
404 | 1294,1,1,"Gibson, Miss. Dorothy Winifred",female,22,0,1,112378,59.4,,C
405 | 1295,0,1,"Carrau, Mr. Jose Pedro",male,17,0,0,113059,47.1,,S
406 | 1296,1,1,"Frauenthal, Mr. Isaac Gerald",male,43,1,0,17765,27.7208,D40,C
407 | 1297,1,2,"Nourney, Mr. Alfred (Baron von Drachstedt"")""",male,20,0,0,SC/PARIS 2166,13.8625,D38,C
408 | 1298,0,2,"Ware, Mr. William Jeffery",male,23,1,0,28666,10.5,,S
409 | 1299,0,1,"Widener, Mr. George Dunton",male,50,1,1,113503,211.5,C80,C
410 | 1300,1,3,"Riordan, Miss. Johanna Hannah""""",female,,0,0,334915,7.7208,,Q
411 | 1301,0,3,"Peacock, Miss. Treasteall",female,3,1,1,SOTON/O.Q. 3101315,13.775,,S
412 | 1302,0,3,"Naughton, Miss. Hannah",female,,0,0,365237,7.75,,Q
413 | 1303,1,1,"Minahan, Mrs. William Edward (Lillian E Thorpe)",female,37,1,0,19928,90,C78,Q
414 | 1304,0,3,"Henriksson, Miss. Jenny Lovisa",female,28,0,0,347086,7.775,,S
415 | 1305,0,3,"Spector, Mr. Woolf",male,,0,0,A.5. 3236,8.05,,S
416 | 1306,1,1,"Oliva y Ocana, Dona. Fermina",female,39,0,0,PC 17758,108.9,C105,C
417 | 1307,0,3,"Saether, Mr. Simon Sivertsen",male,38.5,0,0,SOTON/O.Q. 3101262,7.25,,S
418 | 1308,0,3,"Ware, Mr. Frederick",male,,0,0,359309,8.05,,S
419 | 1309,1,3,"Peter, Master. Michael J",male,,1,1,2668,22.3583,,C
420 |
```
--------------------------------------------------------------------------------
/supervised/model_framework.py:
--------------------------------------------------------------------------------
```python
1 | import copy
2 | import gc
3 | import json
4 | import logging
5 | import os
6 | import time
7 | import uuid
8 |
9 | import numpy as np
10 | import pandas as pd
11 |
12 | from supervised.algorithms.factory import AlgorithmFactory
13 | from supervised.algorithms.registry import (
14 | BINARY_CLASSIFICATION,
15 | MULTICLASS_CLASSIFICATION,
16 | REGRESSION,
17 | AlgorithmsRegistry,
18 | )
19 | from supervised.callbacks.callback_list import CallbackList
20 | from supervised.exceptions import AutoMLException
21 | from supervised.preprocessing.preprocessing import Preprocessing
22 | from supervised.utils.additional_metrics import AdditionalMetrics
23 | from supervised.utils.config import LOG_LEVEL
24 | from supervised.utils.jsonencoder import MLJSONEncoder
25 | from supervised.utils.metric import Metric
26 | from supervised.validation.validation_step import ValidationStep
27 |
28 | logger = logging.getLogger(__name__)
29 | logger.setLevel(LOG_LEVEL)
30 |
31 | import joblib
32 |
33 | from supervised.tuner.optuna.tuner import OptunaTuner
34 | from supervised.utils.learning_curves import LearningCurves
35 |
36 |
37 | class ModelFramework:
38 | def __init__(self, params, callbacks=[]):
39 | logger.debug("ModelFramework.__init__")
40 | self.uid = str(uuid.uuid4())
41 |
42 | for i in ["learner", "validation_strategy"]: # mandatory parameters
43 | if i not in params:
44 | msg = "Missing {0} parameter in ModelFramework params".format(i)
45 | logger.error(msg)
46 | raise ValueError(msg)
47 |
48 | self.params = params
49 | self.callbacks = CallbackList(callbacks)
50 |
51 | self._name = params.get("name", "model")
52 | self.additional_params = params.get("additional")
53 | self.preprocessing_params = params.get("preprocessing")
54 | self.validation_params = params.get("validation_strategy")
55 | self.learner_params = params.get("learner")
56 |
57 | self._ml_task = params.get("ml_task")
58 | self._explain_level = params.get("explain_level")
59 | self._is_stacked = params.get("is_stacked", False)
60 |
61 | self.validation = None
62 | self.preprocessings = []
63 | self.learners = []
64 |
65 | self.train_time = None
66 | self.final_loss = None
67 | self.metric_name = None
68 | self.oof_predictions = None
69 | self._additional_metrics = None
70 | self._threshold = None # used only for binary classifiers
71 | self._max_time_for_learner = params.get("max_time_for_learner", 3600)
72 | self._oof_predictions_fname = None
73 | self._single_prediction_time = None # prediction time on single sample
74 | self._optuna_time_budget = params.get("optuna_time_budget")
75 | self._optuna_init_params = params.get("optuna_init_params", {})
76 | self._optuna_verbose = params.get("optuna_verbose", True)
77 |
78 | self._fairness_metric = params.get("fairness_metric")
79 | self._fairness_threshold = params.get("fairness_threshold")
80 | self._privileged_groups = params.get("privileged_groups", [])
81 | self._underprivileged_groups = params.get("underprivileged_groups", [])
82 | self._fairness_optimization = params.get("fairness_optimization")
83 | self._is_fair = None
84 |
85 | # the automl random state from AutoML constructor, used in Optuna optimizer
86 | self._automl_random_state = params.get("automl_random_state", 42)
87 |
88 | def get_train_time(self):
89 | return self.train_time
90 |
91 | def predictions(
92 | self,
93 | learner,
94 | preproces,
95 | X_train,
96 | y_train,
97 | sample_weight,
98 | sensitive_features,
99 | X_validation,
100 | y_validation,
101 | sample_weight_validation,
102 | sensitive_features_validation,
103 | ):
104 | y_train_true = y_train
105 | y_train_predicted = learner.predict(X_train)
106 | y_validation_true = y_validation
107 | y_validation_predicted = learner.predict(X_validation)
108 |
109 | y_train_true = preproces.inverse_scale_target(y_train_true)
110 | y_train_predicted = preproces.inverse_scale_target(y_train_predicted)
111 | y_validation_true = preproces.inverse_scale_target(y_validation_true)
112 | y_validation_predicted = preproces.inverse_scale_target(y_validation_predicted)
113 |
114 | y_validation_columns = []
115 | if self._ml_task == MULTICLASS_CLASSIFICATION:
116 | # y_train_true = preproces.inverse_categorical_target(y_train_true)
117 | # y_validation_true = preproces.inverse_categorical_target(y_validation_true)
118 | # get columns, omit the last one (it is label)
119 | y_validation_columns = preproces.prepare_target_labels(
120 | y_validation_predicted
121 | ).columns.tolist()[:-1]
122 | elif self._ml_task == BINARY_CLASSIFICATION:
123 | class_names = self.preprocessings[-1].get_target_class_names()
124 | y_validation_columns = "prediction"
125 | if not ("0" in class_names and "1" in class_names):
126 | y_validation_columns = (
127 | f"prediction_0_for_{class_names[0]}_1_for_{class_names[1]}"
128 | )
129 | else:
130 | y_validation_columns = "prediction"
131 |
132 | return {
133 | "y_train_true": y_train_true,
134 | "y_train_predicted": y_train_predicted,
135 | "sample_weight": sample_weight,
136 | "sensitive_features": sensitive_features,
137 | "y_validation_true": y_validation_true,
138 | "y_validation_predicted": y_validation_predicted,
139 | "sample_weight_validation": sample_weight_validation,
140 | "sensitive_features_validation": sensitive_features_validation,
141 | "validation_index": X_validation.index,
142 | "validation_columns": y_validation_columns,
143 | }
144 |
145 | def train(self, results_path, model_subpath):
146 | logger.debug(f"ModelFramework.train {self.learner_params.get('model_type')}")
147 |
148 | start_time = time.time()
149 | np.random.seed(self.learner_params["seed"])
150 |
151 | optuna_tuner = None
152 | if self._optuna_time_budget is not None and OptunaTuner.is_optimizable(
153 | self.learner_params.get("model_type", "")
154 | ):
155 | optuna_tuner = OptunaTuner(
156 | results_path,
157 | ml_task=self._ml_task,
158 | eval_metric=self.get_metric(),
159 | time_budget=self._optuna_time_budget,
160 | init_params=self._optuna_init_params,
161 | verbose=self._optuna_verbose,
162 | n_jobs=self.learner_params.get("n_jobs", -1),
163 | random_state=self._automl_random_state,
164 | )
165 |
166 | self.validation = ValidationStep(self.validation_params)
167 |
168 | repeats = self.validation.get_repeats()
169 | for repeat in range(repeats):
170 | for k_fold in range(self.validation.get_n_splits()):
171 | train_data, validation_data = self.validation.get_split(k_fold, repeat)
172 | logger.debug(
173 | "Data split, train X:{} y:{}, validation X:{}, y:{}".format(
174 | train_data["X"].shape,
175 | train_data["y"].shape,
176 | validation_data["X"].shape,
177 | validation_data["y"].shape,
178 | )
179 | )
180 | if "sample_weight" in train_data:
181 | logger.debug("Sample weight available during the training.")
182 |
183 | # the proprocessing is done at every validation step
184 | self.preprocessings += [
185 | Preprocessing(
186 | self.preprocessing_params, self.get_name(), k_fold, repeat
187 | )
188 | ]
189 |
190 | X_train, y_train, sample_weight = self.preprocessings[
191 | -1
192 | ].fit_and_transform(
193 | train_data["X"], train_data["y"], train_data.get("sample_weight")
194 | )
195 | (
196 | X_validation,
197 | y_validation,
198 | sample_weight_validation,
199 | ) = self.preprocessings[-1].transform(
200 | validation_data["X"],
201 | validation_data["y"],
202 | validation_data.get("sample_weight"),
203 | )
204 |
205 | # skip preprocessing for sensitive features
206 | # TODO: need to add sensitive features in preprocessing because some rows might be skipped (missing target)
207 | # then we need to skip some rows in sensitive features as well
208 | # TODO: drop rows if there is missing data in sensitive feature?
209 |
210 | # get sensitive features from data split
211 | sensitive_features = train_data.get("sensitive_features")
212 | sensitive_features_validation = validation_data.get(
213 | "sensitive_features"
214 | )
215 |
216 | if optuna_tuner is not None:
217 | optuna_start_time = time.time()
218 | self.learner_params = optuna_tuner.optimize(
219 | self.learner_params.get("model_type", ""),
220 | self.params.get("data_type", ""),
221 | X_train,
222 | y_train,
223 | sample_weight,
224 | X_validation,
225 | y_validation,
226 | sample_weight_validation,
227 | self.learner_params,
228 | )
229 | # exclude optuna optimize time from model training
230 | start_time += time.time() - optuna_start_time
231 |
232 | self.learner_params["explain_level"] = self._explain_level
233 | self.learners += [
234 | AlgorithmFactory.get_algorithm(copy.deepcopy(self.learner_params))
235 | ]
236 | learner = self.learners[-1]
237 | learner.set_learner_name(k_fold, repeat, repeats)
238 |
239 | self.callbacks.add_and_set_learner(learner)
240 | self.callbacks.on_learner_train_start()
241 |
242 | log_to_file = os.path.join(
243 | results_path, model_subpath, f"{learner.name}_training.log"
244 | )
245 |
246 | for i in range(learner.max_iters):
247 | self.callbacks.on_iteration_start()
248 |
249 | learner.fit(
250 | X_train,
251 | y_train,
252 | sample_weight,
253 | X_validation,
254 | y_validation,
255 | sample_weight_validation,
256 | log_to_file,
257 | self._max_time_for_learner,
258 | )
259 |
260 | if self.params.get("injected_sample_weight", False):
261 | # print("Dont use sample weight in model evaluation")
262 | sample_weight = None
263 | sample_weight_validation = None
264 |
265 | self.callbacks.on_iteration_end(
266 | {"iter_cnt": i},
267 | self.predictions(
268 | learner,
269 | self.preprocessings[-1],
270 | X_train,
271 | y_train,
272 | sample_weight,
273 | sensitive_features,
274 | X_validation,
275 | y_validation,
276 | sample_weight_validation,
277 | sensitive_features_validation,
278 | ),
279 | )
280 |
281 | if learner.stop_training:
282 | break
283 | learner.update({"step": i})
284 |
285 | # end of learner iters loop
286 | self.callbacks.on_learner_train_end()
287 |
288 | model_path = os.path.join(results_path, model_subpath)
289 | learner.interpret(
290 | X_train,
291 | y_train,
292 | X_validation,
293 | y_validation,
294 | model_file_path=model_path,
295 | learner_name=learner.name,
296 | class_names=self.preprocessings[-1].get_target_class_names(),
297 | metric_name=self.get_metric_name(),
298 | ml_task=self._ml_task,
299 | explain_level=self._explain_level,
300 | )
301 |
302 | # save learner and free the memory
303 | p = os.path.join(model_path, learner.get_fname())
304 | learner.save(p)
305 | del learner.model
306 | learner.model = None
307 | # end of learner training
308 |
309 | # clear data
310 | del X_train
311 | del y_train
312 | del X_validation
313 | del y_validation
314 |
315 | if sample_weight is not None:
316 | del sample_weight
317 | del train_data["sample_weight"]
318 | if sample_weight_validation is not None:
319 | del sample_weight_validation
320 | del validation_data["sample_weight"]
321 |
322 | del train_data["X"]
323 | del train_data["y"]
324 | del validation_data["X"]
325 | del validation_data["y"]
326 | del train_data
327 | del validation_data
328 |
329 | gc.collect()
330 |
331 | # end of validation loop
332 | self.callbacks.on_framework_train_end()
333 | # self.get_additional_metrics()
334 | self._additional_metrics = self.get_additional_metrics()
335 |
336 | self.train_time = time.time() - start_time
337 | logger.debug("ModelFramework end of training")
338 |
339 | def release_learners(self):
340 | for learner in self.learners:
341 | if learner.model is not None:
342 | del learner.model
343 | learner.model = None
344 |
345 | def get_metric_name(self):
346 | if self.metric_name is not None:
347 | return self.metric_name
348 | early_stopping = self.callbacks.get("early_stopping")
349 | if early_stopping is None:
350 | return None
351 | self.metric_name = early_stopping.metric.name
352 | return early_stopping.metric.name
353 |
354 | def get_metric(self):
355 | early_stopping = self.callbacks.get("early_stopping")
356 | if early_stopping:
357 | return early_stopping.metric
358 | return Metric({"name": self.get_metric_name()})
359 |
360 | def get_out_of_folds(self):
361 | if self.oof_predictions is not None:
362 | return self.oof_predictions.copy(deep=True)
363 |
364 | if self._oof_predictions_fname is not None:
365 | self.oof_predictions = pd.read_csv(self._oof_predictions_fname)
366 | return self.oof_predictions.copy(deep=True)
367 |
368 | early_stopping = self.callbacks.get("early_stopping")
369 | if early_stopping is None:
370 | return None
371 | self.oof_predictions = early_stopping.best_y_oof
372 |
373 | ###############################################################
374 | # in case of one-hot coded multiclass target
375 | target_cols = [
376 | c for c in self.oof_predictions.columns.tolist() if "target" in c
377 | ]
378 | if len(target_cols) > 1:
379 | target = self.oof_predictions[target_cols[0]].copy()
380 | target.name = "target"
381 | for i, t in enumerate(target_cols):
382 | target[self.oof_predictions[t] == 1] = i
383 | self.oof_predictions.drop(target_cols, axis=1, inplace=True)
384 |
385 | self.oof_predictions.insert(0, "target", np.array(target))
386 |
387 | return early_stopping.best_y_oof
388 |
389 | def get_final_loss(self):
390 | if self.final_loss is not None:
391 | return self.final_loss
392 | early_stopping = self.callbacks.get("early_stopping")
393 | if early_stopping is None:
394 | return None
395 | self.final_loss = early_stopping.final_loss
396 | return early_stopping.final_loss
397 |
398 | """
399 | def get_metric_logs(self):
400 | metric_logger = self.callbacks.get("metric_logger")
401 | if metric_logger is None:
402 | return None
403 | return metric_logger.loss_values
404 | """
405 |
406 | def get_type(self):
407 | return self.learner_params.get("model_type")
408 |
409 | def get_name(self):
410 | return self._name
411 |
412 | def involved_model_names(self):
413 | """Returns the list of all models involved in the current model.
414 | For single model, it returns the list with the name of the model.
415 | For ensemble model, it returns the list with the name of the ensemble and all internal models
416 | (used to build ensemble).
417 | For single model but trained on stacked data, it returns the list with the name of the model
418 | (names of models used in stacking are not included)."""
419 | return [self._name]
420 |
421 | def is_valid(self):
422 | """is_valid is used in Ensemble to check if it has more than 1 model in it.
423 | If Ensemble has only 1 model in it, then Ensemble shouldn't be used as best model
424 | """
425 | return True
426 |
427 | def is_fast_enough(self, max_single_prediction_time):
428 | # dont need to check
429 | if max_single_prediction_time is None:
430 | return True
431 |
432 | # no iformation about prediction time
433 | if self._single_prediction_time is None:
434 | return True
435 |
436 | return self._single_prediction_time < max_single_prediction_time
437 |
438 | def predict(self, X):
439 | logger.debug("ModelFramework.predict")
440 |
441 | if self.learners is None or len(self.learners) == 0:
442 | raise Exception("Learnes are not initialized")
443 | # run predict on all learners and return the average
444 | y_predicted = None # np.zeros((X.shape[0],))
445 | for ind, learner in enumerate(self.learners):
446 | # preprocessing goes here
447 | X_data, _, _ = self.preprocessings[ind].transform(X.copy(), None)
448 | y_p = learner.predict(X_data)
449 | y_p = self.preprocessings[ind].inverse_scale_target(y_p)
450 |
451 | y_predicted = y_p if y_predicted is None else y_predicted + y_p
452 |
453 | y_predicted_average = y_predicted / float(len(self.learners))
454 |
455 | y_predicted_final = self.preprocessings[0].prepare_target_labels(
456 | y_predicted_average
457 | )
458 |
459 | return y_predicted_final
460 |
461 | def get_additional_metrics(self):
462 | if self._additional_metrics is None:
463 | logger.debug("Compute additional metrics")
464 | # 'target' - the target after processing used for model training
465 | # 'prediction' - out of folds predictions of the model
466 | oof_predictions = self.get_out_of_folds()
467 | prediction_cols = [c for c in oof_predictions.columns if "prediction" in c]
468 | target_cols = [c for c in oof_predictions.columns if "target" in c]
469 |
470 | target = oof_predictions[target_cols]
471 |
472 | oof_preds = None
473 | if self._ml_task == MULTICLASS_CLASSIFICATION:
474 | oof_preds = self.preprocessings[0].prepare_target_labels(
475 | oof_predictions[prediction_cols].values
476 | )
477 | else:
478 | oof_preds = oof_predictions[prediction_cols]
479 |
480 | sample_weight = None
481 | if "sample_weight" in oof_predictions.columns:
482 | sample_weight = oof_predictions["sample_weight"]
483 |
484 | sensitive_features = None
485 | sensitive_cols = [c for c in oof_predictions.columns if "sensitive" in c]
486 | if sensitive_cols:
487 | sensitive_features = oof_predictions[sensitive_cols]
488 |
489 | self._additional_metrics = AdditionalMetrics.compute(
490 | target,
491 | oof_preds,
492 | sample_weight,
493 | self._ml_task,
494 | sensitive_features,
495 | self._fairness_metric
496 | if self._ml_task != REGRESSION
497 | else f"{self._fairness_metric}@{self.get_metric_name()}",
498 | self._fairness_threshold,
499 | self._privileged_groups,
500 | self._underprivileged_groups,
501 | self._fairness_optimization,
502 | )
503 | if self._ml_task == BINARY_CLASSIFICATION:
504 | self._threshold = float(self._additional_metrics["threshold"])
505 | return self._additional_metrics
506 |
507 | def get_sensitive_features_names(self):
508 | metrics = self.get_additional_metrics()
509 | fm = metrics.get("fairness_metrics", {})
510 | return [i for i in list(fm.keys()) if i != "fairness_optimization"]
511 |
512 | def get_fairness_metric(self, col_name):
513 | metrics = self.get_additional_metrics()
514 | fm = metrics.get("fairness_metrics", {})
515 | return fm.get(col_name, {}).get("fairness_metric_value")
516 |
517 | def get_fairness_optimization(self):
518 | metrics = self.get_additional_metrics()
519 | fm = metrics.get("fairness_metrics", {})
520 | return fm.get("fairness_optimization", {})
521 |
522 | def get_worst_fairness(self):
523 | # We have fairness metrics per sensitive feature.
524 | # The worst fairness metric is:
525 | # - for ratio metrics, the lowest fairness value from all sensitive features
526 | # - for difference metrics, the highest fairness value from all sensitive features
527 | # It is needed as bias mitigation stop criteria.
528 |
529 | metrics = self.get_additional_metrics()
530 |
531 | fm = metrics.get("fairness_metrics", {})
532 | worst_value = None
533 | for col_name, values in fm.items():
534 | if col_name == "fairness_optimization":
535 | continue
536 | if "ratio" in self._fairness_metric.lower():
537 | if worst_value is None:
538 | worst_value = values.get("fairness_metric_value", 0)
539 | else:
540 | worst_value = min(
541 | worst_value, values.get("fairness_metric_value", 0)
542 | )
543 | else:
544 | if worst_value is None:
545 | worst_value = values.get("fairness_metric_value", 1)
546 | else:
547 | worst_value = max(
548 | worst_value, values.get("fairness_metric_value", 1)
549 | )
550 |
551 | return worst_value
552 |
553 | def get_best_fairness(self):
554 | # We have fairness metrics per sensitive feature.
555 | # The best fairness metric is:
556 | # - for ratio metrics, the highest fairness value from all sensitive features
557 | # - for difference metrics, the lowest fairness value from all sensitive features
558 | # It is needed as bias mitigation stop criteria.
559 |
560 | metrics = self.get_additional_metrics()
561 | fm = metrics.get("fairness_metrics", {})
562 | best_value = None
563 | for col_name, values in fm.items():
564 | if col_name == "fairness_optimization":
565 | continue
566 | if "ratio" in self._fairness_metric.lower():
567 | if best_value is None:
568 | best_value = values.get("fairness_metric_value", 0)
569 | else:
570 | best_value = max(best_value, values.get("fairness_metric_value", 0))
571 | else:
572 | if best_value is None:
573 | best_value = values.get("fairness_metric_value", 1)
574 | else:
575 | best_value = min(best_value, values.get("fairness_metric_value", 1))
576 |
577 | return best_value
578 |
579 | def is_fair(self):
580 | if self._is_fair is not None:
581 | return self._is_fair
582 | metrics = self.get_additional_metrics()
583 | fm = metrics.get("fairness_metrics", {})
584 | for col, m in fm.items():
585 | if col == "fairness_optimization":
586 | continue
587 | if not m.get("is_fair", True):
588 | self._is_fair = False
589 | return False
590 | self._is_fair = True
591 | return False
592 |
593 | def save(self, results_path, model_subpath):
594 | start_time = time.time()
595 | model_path = os.path.join(results_path, model_subpath)
596 | logger.info(f"Save the model {model_path}")
597 |
598 | type_of_predictions = (
599 | "validation" if "k_folds" not in self.validation_params else "out_of_folds"
600 | )
601 | predictions_fname = os.path.join(
602 | model_subpath, f"predictions_{type_of_predictions}.csv"
603 | )
604 | self._oof_predictions_fname = os.path.join(results_path, predictions_fname)
605 | predictions = self.get_out_of_folds()
606 | predictions.to_csv(self._oof_predictions_fname, index=False)
607 |
608 | saved = [os.path.join(model_subpath, l.get_fname()) for l in self.learners]
609 |
610 | with open(os.path.join(model_path, "framework.json"), "w") as fout:
611 | preprocessing = [p.to_json() for p in self.preprocessings]
612 | learners_params = [learner.get_params() for learner in self.learners]
613 |
614 | desc = {
615 | "uid": self.uid,
616 | "name": self._name,
617 | "preprocessing": preprocessing,
618 | "learners": learners_params,
619 | "params": self.params,
620 | "saved": saved,
621 | "predictions_fname": predictions_fname,
622 | "metric_name": self.get_metric_name(),
623 | "final_loss": self.get_final_loss(),
624 | "train_time": self.get_train_time(),
625 | "is_stacked": self._is_stacked,
626 | "joblib_version": joblib.__version__,
627 | }
628 | desc["final_loss"] = str(desc["final_loss"])
629 | if self._threshold is not None:
630 | desc["threshold"] = self._threshold
631 | if self._single_prediction_time is not None:
632 | desc["single_prediction_time"] = self._single_prediction_time
633 | fout.write(json.dumps(desc, indent=4, cls=MLJSONEncoder))
634 |
635 | learning_curve_metric = self.learners[0].get_metric_name()
636 | if learning_curve_metric is None:
637 | learning_curve_metric = self.get_metric_name()
638 |
639 | LearningCurves.plot(
640 | [l.name for l in self.learners],
641 | learning_curve_metric,
642 | model_path,
643 | trees_in_iteration=self.additional_params.get("trees_in_step"),
644 | )
645 |
646 | # call additional metics just to be sure they are computed
647 | self._additional_metrics = self.get_additional_metrics()
648 |
649 | AdditionalMetrics.save(
650 | self._additional_metrics, self._ml_task, self.model_markdown(), model_path
651 | )
652 |
653 | with open(os.path.join(model_path, "status.txt"), "w") as fout:
654 | fout.write("ALL OK!")
655 | # I'm adding save time to total train time
656 | # there is always save after the training
657 | self.train_time += time.time() - start_time
658 |
659 | def model_markdown(self):
660 | long_name = AlgorithmsRegistry.get_long_name(
661 | self._ml_task, self.learner_params["model_type"]
662 | )
663 | short_name = self.learner_params["model_type"]
664 | desc = f"# Summary of {self.get_name()}\n\n"
665 |
666 | desc += "[<< Go back](../README.md)\n\n"
667 |
668 | if long_name == short_name:
669 | desc += f"\n## {short_name}\n"
670 | else:
671 | desc += f"\n## {long_name} ({short_name})\n"
672 | for k, v in self.learner_params.items():
673 | if k in ["model_type", "ml_task", "seed"]:
674 | continue
675 | desc += f"- **{k}**: {v}\n"
676 | desc += "\n## Validation\n"
677 | for k, v in self.validation_params.items():
678 | if "path" not in k:
679 | desc += f" - **{k}**: {v}\n"
680 | desc += "\n## Optimized metric\n"
681 | desc += f"{self.get_metric_name()}\n"
682 | desc += "\n## Training time\n"
683 | desc += f"\n{np.round(self.train_time,1)} seconds\n"
684 | return desc
685 |
686 | @staticmethod
687 | def load(results_path, model_subpath, lazy_load=True):
688 | model_path = os.path.join(results_path, model_subpath)
689 | logger.info(f"Loading model framework from {model_path}")
690 |
691 | with open(os.path.join(model_path, "framework.json")) as file:
692 | json_desc = json.load(file)
693 |
694 | joblib_version_computer = joblib.__version__
695 | joblib_version_framework = json_desc.get("joblib_version")
696 |
697 | if (
698 | joblib_version_framework is not None
699 | and joblib_version_computer != joblib_version_framework
700 | ):
701 | raise AutoMLException(
702 | f"Joblib version mismatch. Computer: {joblib_version_computer}, Framework: {joblib_version_framework}. Change to Framework version!"
703 | )
704 |
705 | mf = ModelFramework(json_desc["params"])
706 | mf.uid = json_desc.get("uid", mf.uid)
707 | mf._name = json_desc.get("name", mf._name)
708 | mf._threshold = json_desc.get("threshold")
709 | mf.train_time = json_desc.get("train_time", mf.train_time)
710 | mf.final_loss = json_desc.get("final_loss", mf.final_loss)
711 | mf.metric_name = json_desc.get("metric_name", mf.metric_name)
712 | mf._is_stacked = json_desc.get("is_stacked", mf._is_stacked)
713 | mf._single_prediction_time = json_desc.get(
714 | "single_prediction_time", mf._single_prediction_time
715 | )
716 | predictions_fname = json_desc.get("predictions_fname")
717 | if predictions_fname is not None:
718 | mf._oof_predictions_fname = os.path.join(results_path, predictions_fname)
719 |
720 | mf.learners = []
721 | for learner_desc, learner_subpath in zip(
722 | json_desc.get("learners"), json_desc.get("saved")
723 | ):
724 | learner_path = os.path.join(results_path, learner_subpath)
725 | l = AlgorithmFactory.load(learner_desc, learner_path, lazy_load)
726 | mf.learners += [l]
727 |
728 | mf.preprocessings = []
729 | for p in json_desc.get("preprocessing"):
730 | ps = Preprocessing()
731 | ps.from_json(p, results_path)
732 | mf.preprocessings += [ps]
733 |
734 | return mf
735 |
```
--------------------------------------------------------------------------------
/tests/data/CrimeData/cities.json:
--------------------------------------------------------------------------------
```json
1 | [
2 | {
3 | "city": "NewYork",
4 | "latitude": 40.7127837,
5 | "longitude": -74.0059413,
6 | "state": "NY"
7 | },
8 | {
9 | "city":"Greenville",
10 | "latitude":35.6127,
11 | "longitude":-77.3664,
12 | "state":"NC"
13 | },
14 | {
15 | "city" : "Waco",
16 | "latitude":31.5493,
17 | "longitude":-97.1467,
18 | "state":"TX"
19 | },
20 | {
21 | "city":"Richmond",
22 | "latitude":37.9358,
23 | "longitude":-122.3477,
24 | "state":"CA"
25 | },
26 | {
27 | "city": "LosAngeles",
28 | "latitude": 34.0522342,
29 | "longitude": -118.2436849,
30 | "state": "CA"
31 | },
32 | {
33 | "city":"Knoxville",
34 | "latitude":35.9606,
35 | "longitude":-83.9207,
36 | "state":"TN"
37 | },
38 | {
39 | "city": "Chicago",
40 | "latitude": 41.8781136,
41 | "longitude": -87.6297982,
42 | "state": "IL"
43 | },
44 | {
45 | "city" : "Tukwila",
46 | "latitude":47.4740,
47 | "longitude":-122.2610,
48 | "state":"WA"
49 |
50 | },
51 | {
52 | "city" : "Dania",
53 | "latitude":26.0523,
54 | "longitude":-80.1439,
55 | "state":"FL"
56 |
57 | },
58 | {
59 | "city": "Houston",
60 | "latitude": 29.7604267,
61 | "longitude": -95.3698028,
62 | "state": "TX"
63 | },
64 | {
65 | "city": "Philadelphia",
66 | "latitude": 39.9525839,
67 | "longitude": -75.1652215,
68 | "state": "PA"
69 | },
70 | {
71 | "city": "Phoenix",
72 | "latitude": 33.4483771,
73 | "longitude": -112.0740373,
74 | "state": "AZ"
75 | },
76 | {
77 | "city": "San Antonio",
78 | "latitude": 29.4241219,
79 | "longitude": -98.49362819999999,
80 | "state": "TX"
81 | },
82 | {
83 | "city": "San Diego",
84 | "latitude": 32.715738,
85 | "longitude": -117.1610838,
86 | "state": "CA"
87 | },
88 | {
89 | "city": "Dallas",
90 | "latitude": 32.7766642,
91 | "longitude": -96.79698789999999,
92 | "state": "TX"
93 | },
94 | {
95 | "city": "San Jose",
96 | "latitude": 37.3382082,
97 | "longitude": -121.8863286,
98 | "state": "CA"
99 | },
100 | {
101 | "city": "Austin",
102 | "latitude": 30.267153,
103 | "longitude": -97.7430608,
104 | "state": "TX"
105 | },
106 | {
107 | "city": "Indianapolis",
108 | "latitude": 39.768403,
109 | "longitude": -86.158068,
110 | "state": "IN"
111 | },
112 | {
113 | "city": "Jacksonville",
114 | "latitude": 30.3321838,
115 | "longitude": -81.65565099999999,
116 | "state": "FL"
117 | },
118 | {
119 | "city": "SanFrancisco",
120 | "latitude": 37.7749295,
121 | "longitude": -122.4194155,
122 | "state": "CA"
123 | },
124 | {
125 | "city":"SantaFeSprings",
126 | "latitude":33.9472,
127 | "longitude":-118.0853,
128 | "state": "CA"
129 | },
130 | {
131 | "city":"LosBanos",
132 | "latitude":37.0583,
133 | "longitude":-120.8499,
134 | "state":"CA"
135 | },
136 | {
137 | "city": "Columbus",
138 | "latitude": 39.9611755,
139 | "longitude": -82.99879419999999,
140 | "state": "OH"
141 | },
142 | {
143 | "city": "Charlotte",
144 | "latitude": 35.2270869,
145 | "longitude": -80.8431267,
146 | "state": "NC"
147 | },
148 | {
149 | "city": "Fort Worth",
150 | "latitude": 32.7554883,
151 | "longitude": -97.3307658,
152 | "state": "TX"
153 | },
154 | {
155 | "city": "Detroit",
156 | "latitude": 42.331427,
157 | "longitude": -83.0457538,
158 | "state": "MI"
159 | },
160 | {
161 | "city": "El Paso",
162 | "latitude": 31.7775757,
163 | "longitude": -106.4424559,
164 | "state": "TX"
165 | },
166 | {
167 | "city": "Memphis",
168 | "latitude": 35.1495343,
169 | "longitude": -90.0489801,
170 | "state": "TN"
171 | },
172 | {
173 | "city": "Seattle",
174 | "latitude": 47.6062095,
175 | "longitude": -122.3320708,
176 | "state": "WA"
177 | },
178 | {
179 | "city": "Denver",
180 | "latitude": 39.7392358,
181 | "longitude": -104.990251,
182 | "state": "CO"
183 | },
184 | {
185 | "city": "Washington",
186 | "latitude": 38.9071923,
187 | "longitude": -77.0368707,
188 | "state": "DC"
189 | },
190 | {
191 | "city":"Anniston",
192 | "latitude":33.6598,
193 | "longitude":-85.8316,
194 | "state":"AL"
195 | },
196 | {
197 | "city": "Boston",
198 | "latitude": 42.3600825,
199 | "longitude": -71.0588801,
200 | "state": "MA"
201 | },
202 | {
203 | "city": "Nashville-Davidson",
204 | "latitude": 36.1626638,
205 | "longitude": -86.7816016,
206 | "state": "TN"
207 | },
208 | {
209 | "city": "Baltimore",
210 | "latitude": 39.2903848,
211 | "longitude": -76.6121893,
212 | "state": "MD"
213 | },
214 | {
215 | "city": "Oklahoma City",
216 | "latitude": 35.4675602,
217 | "longitude": -97.5164276,
218 | "state": "OK"
219 | },
220 | {
221 | "city": "Portland",
222 | "latitude": 45.5230622,
223 | "longitude": -122.6764816,
224 | "state": "OR"
225 | },
226 | {
227 | "city": "Las Vegas",
228 | "latitude": 36.1699412,
229 | "longitude": -115.1398296,
230 | "state": "NV"
231 | },
232 | {
233 | "city": "Milwaukee",
234 | "latitude": 43.0389025,
235 | "longitude": -87.9064736,
236 | "state": "WI"
237 | },
238 | {
239 | "city": "Albuquerque",
240 | "latitude": 35.0853336,
241 | "longitude": -106.6055534,
242 | "state": "NM"
243 | },
244 | {
245 | "city": "Tucson",
246 | "latitude": 32.2217429,
247 | "longitude": -110.926479,
248 | "state": "AZ"
249 | },
250 | {
251 | "city":"SouthElMonte",
252 | "latitude":34.0520,
253 | "longitude":-118.0467,
254 | "state":"CA"
255 | },
256 | {
257 | "city":"GlensFalls",
258 | "latitude":43.3095,
259 | "longitude":-73.6440,
260 | "state":"NY"
261 | },
262 | {
263 | "city":"Cudahy",
264 | "latitude":33.9606,
265 | "longitude":-118.1853,
266 | "state":"CA"
267 | },
268 | {
269 | "city": "Fresno",
270 | "latitude": 36.7468422,
271 | "longitude": -119.7725868,
272 | "state": "CA"
273 | },
274 | {
275 | "city": "LongBeach",
276 | "latitude": 33.7700504,
277 | "longitude": -118.1937395,
278 | "state": "CA"
279 | },
280 | {
281 | "city": "KansasCity",
282 | "latitude": 39.0997265,
283 | "longitude": -94.5785667,
284 | "state": "MO"
285 | },
286 | {
287 | "city": "Omaha",
288 | "latitude": 41.2523634,
289 | "longitude": -95.99798829999999,
290 | "state": "NE"
291 | },
292 | {
293 | "city": "Raleigh",
294 | "latitude": 35.7795897,
295 | "longitude": -78.6381787,
296 | "state": "NC"
297 | },
298 | {
299 | "city": "Miami",
300 | "latitude": 25.7616798,
301 | "longitude": -80.1917902,
302 | "state": "FL"
303 | },
304 | {
305 | "city": "Oakland",
306 | "latitude": 37.8043637,
307 | "longitude": -122.2711137,
308 | "state": "CA"
309 | },
310 | {
311 | "city": "Minneapolis",
312 | "latitude": 44.977753,
313 | "longitude": -93.2650108,
314 | "state": "MN"
315 | },
316 | {
317 | "city":"Pleasantville",
318 | "latitude":39.3898,
319 | "longitude":-74.5240,
320 | "state":"NJ"
321 |
322 | },
323 | {
324 | "city": "Tulsa",
325 | "latitude": 36.1539816,
326 | "longitude": -95.99277500000001,
327 | "state": "OK"
328 | },
329 | {
330 | "city": "Cleveland",
331 | "latitude": 41.49932,
332 | "longitude": -81.6943605,
333 | "state": "OH"
334 | },
335 | {
336 | "city": "Wichita",
337 | "latitude": 37.688889,
338 | "longitude": -97.336111,
339 | "state": "KS"
340 | },
341 | {
342 | "city": "Arlington",
343 | "latitude": 32.735687,
344 | "longitude": -97.10806559999999,
345 | "state": "TX"
346 | },
347 | {
348 | "city": "NewOrleans",
349 | "latitude": 29.95106579999999,
350 | "longitude": -90.0715323,
351 | "state": "LA"
352 | },
353 | {
354 | "city":"Prichard",
355 | "latitude":30.7452,
356 | "longitude":-88.0897,
357 | "state":"AL"
358 | },
359 | {
360 | "city":"Gadsden",
361 | "latitude":34.0143,
362 | "longitude":-86.0066,
363 | "state":"AL"
364 | },
365 | {
366 | "city": "Bakersfield",
367 | "latitude": 35.3732921,
368 | "longitude": -119.0187125,
369 | "state": "CA"
370 | },
371 | {
372 | "city": "Tampa",
373 | "latitude": 27.950575,
374 | "longitude": -82.4571776,
375 | "state": "FL"
376 | },
377 | {
378 | "city": "Anaheim",
379 | "latitude": 33.8352932,
380 | "longitude": -117.9145036,
381 | "state": "CA"
382 | },
383 | {
384 | "city": "St. Louis",
385 | "latitude": 38.6270025,
386 | "longitude": -90.19940419999999,
387 | "state": "MO"
388 | },
389 | {
390 | "city": "Lexington-Fayette",
391 | "latitude": 38.0405837,
392 | "longitude": -84.5037164,
393 | "state": "KY"
394 | },
395 | {
396 | "city": "Pittsburgh",
397 | "latitude": 40.44062479999999,
398 | "longitude": -79.9958864,
399 | "state": "PA"
400 | },
401 | {
402 | "city": "Anchorage",
403 | "latitude": 61.2180556,
404 | "longitude": -149.9002778,
405 | "state": "AK"
406 | },
407 | {
408 | "city": "Cincinnati",
409 | "latitude": 39.1031182,
410 | "longitude": -84.5120196,
411 | "state": "OH"
412 | },
413 | {
414 | "city":"Paducah",
415 | "latitude":37.0834,
416 | "longitude":-88.6000,
417 | "state":"KY"
418 | },
419 | {
420 | "city":"Blytheville",
421 | "latitude":35.9273,
422 | "longitude":-89.9190,
423 | "state":"AR"
424 | },
425 | {
426 | "city": "St. Paul",
427 | "latitude": 44.9537029,
428 | "longitude": -93.0899578,
429 | "state": "MN"
430 | },
431 | {
432 | "city": "Toledo",
433 | "latitude": 41.6639383,
434 | "longitude": -83.55521200000001,
435 | "state": "OH"
436 | },
437 | {
438 | "city":"Tallahassee",
439 | "latitude":30.4383,
440 | "longitude":-84.2807,
441 | "state":"FL"
442 | },
443 | {
444 | "city":"EastLongmeadow",
445 | "latitude":42.0645,
446 | "longitude":-72.5126,
447 | "state":"MA"
448 | },
449 | {
450 | "city": "Newark",
451 | "latitude": 40.735657,
452 | "longitude": -74.1723667,
453 | "state": "NJ"
454 | },
455 | {
456 | "city": "Plano",
457 | "latitude": 33.0198431,
458 | "longitude": -96.6988856,
459 | "state": "TX"
460 | },
461 | {
462 | "city": "Lincoln",
463 | "latitude": 40.8257625,
464 | "longitude": -96.6851982,
465 | "state": "NE"
466 | },
467 | {
468 | "city": "Buffalo",
469 | "latitude": 42.88644679999999,
470 | "longitude": -78.8783689,
471 | "state": "NY"
472 | },
473 | {
474 | "city": "JerseyCity",
475 | "latitude": 40.72815749999999,
476 | "longitude": -74.0776417,
477 | "state": "NJ"
478 | },
479 | {
480 | "city": "Orlando",
481 | "latitude": 28.5383355,
482 | "longitude": -81.3792365,
483 | "state": "FL"
484 | },
485 | {
486 | "city":"DaytonaBeach",
487 | "latitude":29.2108,
488 | "longitude":-81.0228,
489 | "state":"FL"
490 | },
491 | {
492 | "city":"Bridgeporttown",
493 | "latitude":41.1865,
494 | "longitude":-73.1952,
495 | "state":"CT"
496 | },
497 | {
498 | "city":"Youngstown",
499 | "latitude":41.0998,
500 | "longitude":-80.6495,
501 | "state":"OH"
502 | },
503 | {
504 | "city":"Monroe",
505 | "latitude":32.5093,
506 | "longitude":-92.1193,
507 | "state":"LA"
508 | },
509 | {
510 | "city":"Richmond",
511 | "latitude":37.5407,
512 | "longitude":-77.4360,
513 | "state":"VA"
514 | },
515 | {
516 | "city": "St.Petersburg",
517 | "latitude": 27.773056,
518 | "longitude": -82.64,
519 | "state": "FL"
520 | },
521 | {
522 | "city": "Chandler",
523 | "latitude": 33.3061605,
524 | "longitude": -111.8412502,
525 | "state": "AZ"
526 | },
527 | {
528 | "city": "Laredo",
529 | "latitude": 27.5305671,
530 | "longitude": -99.48032409999999,
531 | "state": "TX"
532 | },
533 | {
534 | "city": "Glendale",
535 | "latitude": 33.5386523,
536 | "longitude": -112.1859866,
537 | "state": "AZ"
538 | },
539 | {
540 | "city":"Bessemer",
541 | "latitude":33.4018,
542 | "longitude":-86.9544,
543 | "state":"AL"
544 | },
545 | {
546 | "city":"PlantCity",
547 | "latitude":28.0186,
548 | "longitude":-82.1129,
549 | "state":"FL"
550 | },
551 | {
552 | "city":"CityofOrangetownship",
553 | "latitude":40.7706,
554 | "longitude":-74.2326,
555 | "state":"NJ"
556 | },
557 | {
558 | "city":"Palatka",
559 | "latitude":29.6486,
560 | "longitude":-81.6376,
561 | "state":"FL"
562 | },
563 | {
564 | "city":"Artesia",
565 | "latitude":33.8658,
566 | "longitude":-118.0831,
567 | "state":"CA"
568 | },
569 | {
570 | "city":"Irvingtontownship",
571 | "latitude":40.7263,
572 | "longitude":-74.2286,
573 | "state":"NJ"
574 | },
575 | {
576 | "city":"CollegePark",
577 | "latitude":33.6534,
578 | "longitude":-84.4494,
579 | "state":"GA"
580 | },
581 | {
582 | "city":"Greenwood",
583 | "latitude":34.1954,
584 | "longitude":-82.1618,
585 | "state":"SC"
586 | },
587 | {
588 | "city":"Brunswick",
589 | "latitude":31.1500,
590 | "longitude":-81.4915,
591 | "state":"GA"
592 | },
593 | {
594 | "city":"Spartanburg",
595 | "latitude":34.9496,
596 | "longitude":-81.9320,
597 | "state":"SC"
598 | },
599 | {
600 | "city":"Rome",
601 | "latitude":34.2570,
602 | "longitude":-85.1647,
603 | "state":"GA"
604 | },
605 | {
606 | "city":"PortArthur",
607 | "latitude":29.8850,
608 | "longitude":-93.9399,
609 | "state":"TX"
610 | },
611 | {
612 | "city": "BatonRouge",
613 | "latitude": 30.4582829,
614 | "longitude": -91.1403196,
615 | "state": "LA"
616 | },
617 | {
618 | "city": "SanBernardino",
619 | "latitude": 34.1083449,
620 | "longitude": -117.2897652,
621 | "state": "CA"
622 | },
623 | {
624 | "city":"WestHollywood",
625 | "latitude":34.0900,
626 | "longitude":-118.3617,
627 | "state":"CA"
628 | },
629 |
630 | {
631 | "city": "Birmingham",
632 | "latitude": 33.5206608,
633 | "longitude": -86.80248999999999,
634 | "state": "AL"
635 | },
636 | {
637 | "city":"NewHaventown",
638 | "latitude":41.3083,
639 | "longitude":-72.9279,
640 | "state":"CT"
641 | },
642 | {
643 | "city":"Aurora",
644 | "latitude":39.7294,
645 | "longitude":-104.8319,
646 | "state":"CO"
647 | },
648 | {
649 | "city":"SiouxCity",
650 | "latitude":42.5000,
651 | "longitude":-96.4003,
652 | "state":"IA"
653 |
654 | },
655 | {
656 | "city":"Grenada",
657 | "latitude":33.7690,
658 | "longitude":-89.8084,
659 | "state":"MS"
660 | },
661 | {
662 | "city":"AsburyPark",
663 | "latitude":40.2204,
664 | "longitude":-74.0121,
665 | "state":"NJ"
666 | },
667 | {
668 | "city":"Gretna",
669 | "latitude":29.9146,
670 | "longitude":-90.0540,
671 | "state":"LA"
672 | },
673 | {
674 | "city":"Florence",
675 | "latitude":34.1954,
676 | "longitude":-79.7626,
677 | "state":"SC"
678 |
679 | },
680 | {
681 | "city":"Vernon",
682 | "latitude":34.1545,
683 | "longitude":-99.2651,
684 | "state":"TX"
685 | },
686 | {
687 | "city":"Steubenville",
688 | "latitude":40.3698,
689 | "longitude":-80.6340,
690 | "state":"OH"
691 | },
692 | {
693 | "city":"Riverside",
694 | "latitude":33.9533,
695 | "longitude":-117.3962,
696 | "state":"CA"
697 | },
698 | {
699 | "city":"NewBern",
700 | "latitude":35.1085,
701 | "longitude":-77.0441,
702 | "state":"NC"
703 | },
704 | {
705 | "city":"Opelika",
706 | "latitude":32.6454,
707 | "longitude":-85.3783,
708 | "state":"AL"
709 | },
710 | {
711 | "city":"Wilkinsburgborough",
712 | "latitude":40.4417,
713 | "longitude":-79.8820,
714 | "state":"PA"
715 | },
716 | {
717 | "city":"Stockton",
718 | "latitude":37.9577,
719 | "longitude":-121.2908,
720 | "state":"CA"
721 | },
722 | {
723 | "city":"Newberrytown",
724 | "latitude":34.2746,
725 | "longitude":-81.6187,
726 | "state":"SC"
727 | },
728 | {
729 | "city":"EastPoint",
730 | "latitude":33.6796,
731 | "longitude":-84.4394,
732 | "state":"GA"
733 | },
734 | {
735 | "city":"SouthMiami",
736 | "latitude":25.7051,
737 | "longitude":-80.2908,
738 | "state":"FL"
739 | },
740 | {
741 | "city":"BalchSprings",
742 | "latitude":32.7287,
743 | "longitude":-96.6228,
744 | "state":"TX"
745 | },
746 | {
747 | "city":"Fairfield",
748 | "latitude":33.4859,
749 | "longitude":-86.9119,
750 | "state":"AL"
751 | },
752 | {
753 | "city": "Des Moines",
754 | "latitude": 41.6005448,
755 | "longitude": -93.6091064,
756 | "state": "Iowa"
757 | },
758 | {
759 | "city": "Fayetteville",
760 | "latitude": 35.0526641,
761 | "longitude": -78.87835849999999,
762 | "state": "NC"
763 | },
764 | {
765 | "city": "Tacoma",
766 | "latitude": 47.2528768,
767 | "longitude": -122.4442906,
768 | "state": "WA"
769 | },
770 | {
771 | "city": "LittleRock",
772 | "latitude": 34.7464809,
773 | "longitude": -92.28959479999999,
774 | "state": "AR"
775 | },
776 | {
777 | "city":"Gallup",
778 | "latitude":35.5281,
779 | "longitude":-108.7426,
780 | "state":"NM"
781 | },
782 | {
783 | "city": "Chattanooga",
784 | "latitude": 35.0456297,
785 | "longitude": -85.3096801,
786 | "state": "TN"
787 | },
788 | {
789 | "city": "Bridgeport",
790 | "latitude": 41.1865478,
791 | "longitude": -73.19517669999999,
792 | "state": "CT"
793 | },
794 | {
795 | "city":"LakeWorth",
796 | "latitude":26.6168,
797 | "longitude":-80.0684,
798 | "state":"FL"
799 | },
800 | {
801 | "city": "Paterson",
802 | "latitude": 40.9167654,
803 | "longitude": -74.17181099999999,
804 | "state": "NJ"
805 | },
806 | {
807 | "city":"Douglas",
808 | "latitude":31.5088,
809 | "longitude":-82.8499,
810 | "state":"GA"
811 | },
812 | {
813 | "city": "Dayton",
814 | "latitude": 39.7589478,
815 | "longitude": -84.1916069,
816 | "state": "OH"
817 | },
818 | {
819 | "city": "Columbia",
820 | "latitude": 34.0007104,
821 | "longitude": -81.0348144,
822 | "state": "SC"
823 | },
824 | {
825 | "city":"WestColumbia",
826 | "latitude":33.9935,
827 | "longitude":-81.0740,
828 | "state":"SC"
829 | },
830 | {
831 | "city": "Gainesville",
832 | "latitude": 29.6516344,
833 | "longitude": -82.32482619999999,
834 | "state": "FL"
835 | },
836 | {
837 | "city": "Hartfordtown",
838 | "latitude": 41.76371109999999,
839 | "longitude": -72.6850932,
840 | "state": "CT"
841 | },
842 | {
843 | "city": "Berkeley",
844 | "latitude": 37.8715926,
845 | "longitude": -122.272747,
846 | "state": "CA"
847 | },
848 | {
849 | "city": "Inglewood",
850 | "latitude": 33.9616801,
851 | "longitude": -118.3531311,
852 | "state": "CA"
853 | },
854 | {
855 | "city":"Chester",
856 | "latitude":39.8496,
857 | "longitude":-75.3557,
858 | "state":"PA"
859 | },
860 | {
861 | "city": "Pueblo",
862 | "latitude": 38.2544472,
863 | "longitude": -104.6091409,
864 | "state": "CO"
865 | },
866 | {
867 | "city":"Selma",
868 | "latitude":32.4074,
869 | "longitude":-87.0211,
870 | "state":"AL"
871 | },
872 | {
873 | "city":"Clearlake",
874 | "latitude":38.9582,
875 | "longitude":-122.6264,
876 | "state":"CA"
877 |
878 | },
879 | {
880 | "city":"Paris",
881 | "latitude":33.6609,
882 | "longitude":-95.5555,
883 | "state":"TX"
884 |
885 | },
886 | {
887 | "city":"Newburgh",
888 | "latitude":41.5034,
889 | "longitude":-74.0104,
890 | "state":"NY"
891 | },
892 | {
893 | "city": "PompanoBeach",
894 | "latitude": 26.2378597,
895 | "longitude": -80.1247667,
896 | "state": "FL"
897 | },
898 | {
899 | "city": "NorthCharleston",
900 | "latitude": 32.8546197,
901 | "longitude": -79.9748103,
902 | "state": "SC"
903 | },
904 | {
905 | "city": "Davenport",
906 | "latitude": 41.5236437,
907 | "longitude": -90.5776367,
908 | "state": "IA"
909 | },
910 | {
911 | "city" : "SanPablo",
912 | "latitude":37.9621,
913 | "longitude":122.3455,
914 | "state":"CA"
915 | },
916 | {
917 | "city": "Compton",
918 | "latitude": 33.8958492,
919 | "longitude": -118.2200712,
920 | "state": "CA"
921 | },
922 | {
923 | "city": "SantaMonica",
924 | "latitude": 34.0194543,
925 | "longitude": -118.4911912,
926 | "state": "CA"
927 | },
928 | {
929 | "city": "Lynn",
930 | "latitude": 42.46676300000001,
931 | "longitude": -70.9494938,
932 | "state": "MA"
933 | },
934 | {
935 | "city":"Orangeburg",
936 | "latitude":33.4918,
937 | "longitude":-80.8556,
938 | "state":"SC"
939 | },
940 | {
941 | "city": "MiamiBeach",
942 | "latitude": 25.790654,
943 | "longitude": -80.1300455,
944 | "state": "FL"
945 | },
946 | {
947 | "city": "Greenville",
948 | "latitude": 34.8526,
949 | "longitude": -82.3940,
950 | "state": "SC"
951 | },
952 | {
953 | "city":"Leesburg",
954 | "latitude":28.8108,
955 | "longitude":-81.8779,
956 | "state":"FL"
957 | },
958 | {
959 | "city":"LaPuente",
960 | "latitude":34.0200,
961 | "longitude":-117.9495,
962 | "state":"CA"
963 | },
964 | {
965 | "city":"RockHill",
966 | "latitude":34.9249,
967 | "longitude":-81.0251,
968 | "state":"SC"
969 | },
970 | {
971 | "city": "Hawthorne",
972 | "latitude": 33.9164032,
973 | "longitude": -118.3525748,
974 | "state": "CA"
975 | },
976 | {
977 | "city": "Trenton",
978 | "latitude": 40.2170534,
979 | "longitude": -74.7429384,
980 | "state": "NJ"
981 | },
982 | {
983 | "city": "Gary",
984 | "latitude": 41.5933696,
985 | "longitude": -87.3464271,
986 | "state": "IN"
987 | },
988 | {
989 | "city":"FortPierce",
990 | "latitude":27.4467,
991 | "longitude":-80.3256,
992 | "state":"FL"
993 | },
994 | {
995 | "city": "Lawrence",
996 | "latitude": 42.7070354,
997 | "longitude": -71.1631137,
998 | "state": "MA"
999 | },
1000 | {
1001 | "city":"Saraland",
1002 | "latitude":30.8207,
1003 | "longitude":-88.0706,
1004 | "state":"AL"
1005 | },
1006 | {
1007 | "city": "Bellflower",
1008 | "latitude": 33.8816818,
1009 | "longitude": -118.1170117,
1010 | "state": "CA"
1011 | },
1012 | {
1013 | "city": "Camden",
1014 | "latitude": 39.9259463,
1015 | "longitude": -75.1196199,
1016 | "state": "NJ"
1017 | },
1018 | {
1019 | "city": "Lynwood",
1020 | "latitude": 33.930293,
1021 | "longitude": -118.2114603,
1022 | "state": "CA"
1023 | },
1024 | {
1025 | "city":"Atlanta",
1026 | "latitude":33.7490,
1027 | "longitude":-84.3880,
1028 | "state":"GA"
1029 | },
1030 | {
1031 | "city": "FortMyers",
1032 | "latitude": 26.640628,
1033 | "longitude": -81.8723084,
1034 | "state": "FL"
1035 | },
1036 | {
1037 | "city": "Jackson",
1038 | "latitude": 35.6145169,
1039 | "longitude": -88.81394689999999,
1040 | "state": "TN"
1041 | },
1042 | {
1043 | "city":"Lawndale",
1044 | "latitude":33.8872,
1045 | "longitude":118.3526,
1046 | "state":"CA"
1047 | },
1048 | {
1049 | "city": "EastOrange",
1050 | "latitude": 40.767323,
1051 | "longitude": -74.2048677,
1052 | "state": "NJ"
1053 | },
1054 | {
1055 | "city":"EastChicago",
1056 | "latitude":41.6392,
1057 | "longitude":-87.4548,
1058 | "state":"IN"
1059 | },
1060 | {
1061 | "city": "Homestead",
1062 | "latitude": 25.4687224,
1063 | "longitude": -80.4775569,
1064 | "state": "FL"
1065 | },
1066 | {
1067 | "city":"LaCanadaFlintridge",
1068 | "latitude":34.2068,
1069 | "longitude":-118.2000,
1070 | "state":"CA"
1071 | },
1072 | {
1073 | "city":"DeLand",
1074 | "latitude":29.0283,
1075 | "longitude":-81.3031,
1076 | "state":"FL"
1077 | },
1078 | {
1079 | "city": "NewBraunfels",
1080 | "latitude": 29.7030024,
1081 | "longitude": -98.1244531,
1082 | "state": "TX"
1083 | },
1084 | {
1085 | "city":"Commerce",
1086 | "latitude":34.0006,
1087 | "longitude":-118.1598,
1088 | "state":"CA"
1089 | },
1090 | {
1091 | "city":"Salisbury",
1092 | "latitude":38.3607,
1093 | "longitude":-75.5994,
1094 | "state":"MD"
1095 | },
1096 | {
1097 | "city":"LakeCity",
1098 | "latitude":30.1897,
1099 | "longitude":-82.6393,
1100 | "state":"FL"
1101 | },
1102 | {
1103 | "city": "NorthMiami",
1104 | "latitude": 25.8900949,
1105 | "longitude": -80.1867138,
1106 | "state": "FL"
1107 | },
1108 | {
1109 | "city": "Gardena",
1110 | "latitude": 33.8883487,
1111 | "longitude": -118.3089624,
1112 | "state": "CA"
1113 | },
1114 | {
1115 | "city": "Ocala",
1116 | "latitude": 29.1871986,
1117 | "longitude": -82.14009229999999,
1118 | "state": "FL"
1119 | },
1120 | {
1121 | "city":"Anderson",
1122 | "latitude":34.5034,
1123 | "longitude":82.6501,
1124 | "state":"SC"
1125 | },
1126 | {
1127 | "city": "EastPaloAlto",
1128 | "latitude":37.4688,
1129 | "longitude":-122.1411,
1130 | "state":"CA"
1131 | },
1132 | {
1133 | "city":"Bellingham",
1134 | "latitude":48.7519,
1135 | "longitude":-122.4787,
1136 | "state":"WA"
1137 | },
1138 | {
1139 | "city":"DelrayBeach",
1140 | "latitude":26.4615,
1141 | "longitude":-80.0728,
1142 | "state":"FL"
1143 | },
1144 | {
1145 | "city":"Jackson",
1146 | "latitude":32.2988,
1147 | "longitude":-90.1848,
1148 | "state":"MS"
1149 | },
1150 | {
1151 | "city":"Plainfield",
1152 | "latitude":40.6337,
1153 | "longitude":-74.4074,
1154 | "state":"NJ"
1155 | },
1156 | {
1157 | "city":"Pomona",
1158 | "latitude":34.0551,
1159 | "longitude":-117.7500,
1160 | "state":"CA"
1161 | },
1162 | {
1163 | "city":"HarperWoods",
1164 | "latitude":42.4331,
1165 | "longitude":-82.9241,
1166 | "state":"MI"
1167 | },
1168 | {
1169 | "city":"RichmondHeights",
1170 | "latitude":38.6287,
1171 | "longitude":-90.3196,
1172 | "state":"MO"
1173 | },
1174 | {
1175 | "city":"McAllen",
1176 | "latitude":26.2034,
1177 | "longitude":-98.2300,
1178 | "state":"TX"
1179 | },
1180 | {
1181 | "city":"Greenwood",
1182 | "latitude":33.5162,
1183 | "longitude":-90.1795,
1184 | "state":"MS"
1185 | },
1186 | {
1187 | "city":"CasaGrande",
1188 | "latitude":32.9110,
1189 | "longitude":-111.7734,
1190 | "state":"AZ"
1191 | },
1192 | {
1193 | "city":"CoralGables",
1194 | "latitude":25.7215,
1195 | "longitude":-80.2684,
1196 | "state":"FL"
1197 | },
1198 | {
1199 | "city":"WinterHaven",
1200 | "latitude":28.0222,
1201 | "longitude":-81.7329,
1202 | "state":"FL"
1203 | },
1204 | {
1205 | "city":"Paramusborough",
1206 | "latitude":40.9445,
1207 | "longitude":-74.0754,
1208 | "state":"NJ"
1209 | },
1210 | {
1211 | "city":"Orange",
1212 | "latitude":30.0930,
1213 | "longitude":-93.7366,
1214 | "state":"TX"
1215 | },
1216 | {
1217 | "city":"NorthLittleRock",
1218 | "latitude":34.7695,
1219 | "longitude":-92.2671,
1220 | "state":"AR"
1221 | },
1222 | {
1223 | "city":"Kilgore",
1224 | "latitude":32.3863,
1225 | "longitude":-94.8758,
1226 | "state":"TX"
1227 | },
1228 | {
1229 | "city":"Sanford",
1230 | "latitude":35.4799,
1231 | "longitude":-79.1803,
1232 | "state":"NC"
1233 | },
1234 | {
1235 | "city":"Covington",
1236 | "latitude":33.5968,
1237 | "longitude":-83.8602,
1238 | "state":"GA"
1239 | },
1240 | {
1241 | "city":"Texarkana",
1242 | "latitude":33.4418,
1243 | "longitude":-94.0377,
1244 | "state":"AR"
1245 | },
1246 | {
1247 | "city":"MuskegonHeights",
1248 | "latitude":43.2011,
1249 | "longitude":-86.2389,
1250 | "state":"MI"
1251 | },
1252 | {
1253 | "city":"KeyWest",
1254 | "latitude":24.5551,
1255 | "longitude":-81.7800,
1256 | "state":"FL"
1257 | },
1258 | {
1259 | "city":"Springdale",
1260 | "latitude":39.2870,
1261 | "longitude":-84.4852,
1262 | "state":"OH"
1263 | },
1264 | {
1265 | "city":"HotSprings",
1266 | "latitude":34.5037,
1267 | "longitude":-93.0552,
1268 | "state":"AR"
1269 | },
1270 | {
1271 | "city":"Sunnyside",
1272 | "latitude":46.3237,
1273 | "longitude":-120.0087,
1274 | "state":"WA"
1275 | },
1276 | {
1277 | "city":"Wilson",
1278 | "latitude":35.7213,
1279 | "longitude":-77.9155,
1280 | "state":"NC"
1281 | },
1282 | {
1283 | "city":"Lakeland",
1284 | "latitude":28.0395,
1285 | "longitude":-81.9498,
1286 | "state":"FL"
1287 | },
1288 | {
1289 | "city":"MyrtleBeach",
1290 | "latitude":33.6891,
1291 | "longitude":-78.8867,
1292 | "state":"SC"
1293 | },
1294 | {
1295 | "city": "Paramount",
1296 | "latitude": 33.8894598,
1297 | "longitude": -118.1597911,
1298 | "state": "CA"
1299 | },
1300 | {
1301 | "city":"Henderson",
1302 | "latitude":36.3296,
1303 | "longitude":-78.3992,
1304 | "state":"NC"
1305 | },
1306 | {
1307 | "city": "Sarasota",
1308 | "latitude": 27.3364347,
1309 | "longitude": -82.53065269999999,
1310 | "state": "FL"
1311 | },
1312 | {
1313 | "city":"Bridgeton",
1314 | "latitude":39.4273,
1315 | "longitude":-75.2341,
1316 | "state":"NJ"
1317 | },
1318 | {
1319 | "city": "Harrisburg",
1320 | "latitude": 40.2731911,
1321 | "longitude": -76.8867008,
1322 | "state": "PA"
1323 | },
1324 | {
1325 | "city": "Galveston",
1326 | "latitude": 29.3013479,
1327 | "longitude": -94.7976958,
1328 | "state": "TX"
1329 | },
1330 | {
1331 | "city": "Alexandria",
1332 | "latitude": 31.3112936,
1333 | "longitude": -92.4451371,
1334 | "state": "LA"
1335 | },
1336 | {
1337 | "city":"AtlanticCity",
1338 | "latitude":39.3643,
1339 | "longitude":-74.4229,
1340 | "state":"NJ"
1341 | },
1342 | {
1343 | "city": "Mentor",
1344 | "latitude": 41.6661573,
1345 | "longitude": -81.339552,
1346 | "state": "OH"
1347 | },
1348 | {
1349 | "city": "Mansfield",
1350 | "latitude": 40.75839,
1351 | "longitude": -82.5154471,
1352 | "state": "OH"
1353 | },
1354 | {
1355 | "city": "Barnstabletown",
1356 | "latitude": 41.7003208,
1357 | "longitude": -70.3002024,
1358 | "state": "MA"
1359 | },
1360 | {
1361 | "city": "OaklandPark",
1362 | "latitude": 26.1723065,
1363 | "longitude": -80.1319893,
1364 | "state": "FL"
1365 | },
1366 | {
1367 | "city": "Everett",
1368 | "latitude": 42.40843,
1369 | "longitude": -71.0536625,
1370 | "state": "MA"
1371 | },
1372 | {
1373 | "city": "Covington",
1374 | "latitude": 39.0836712,
1375 | "longitude": -84.5085536,
1376 | "state": "KY"
1377 | },
1378 |
1379 | {
1380 | "city": "Fitchburg",
1381 | "latitude": 42.5834228,
1382 | "longitude": -71.8022955,
1383 | "state": "MA"
1384 | },
1385 | {
1386 | "city": "Hallandale",
1387 | "latitude": 25.9812024,
1388 | "longitude": -80.14837899999999,
1389 | "state": "FL"
1390 | },
1391 | {
1392 | "city": "Lima",
1393 | "latitude": 40.742551,
1394 | "longitude": -84.1052256,
1395 | "state": "OH"
1396 | },
1397 | {
1398 | "city":"FortLauderdale",
1399 | "latitude":26.1224,
1400 | "longitude":-80.1373,
1401 | "state":"FL"
1402 | },
1403 | {
1404 | "city":"SouthSaltLake",
1405 | "latitude":40.7188,
1406 | "longitude":-111.8883,
1407 | "state":"UT"
1408 | },
1409 | {
1410 | "city":"EastLongmeadow",
1411 | "latitude":42.0645,
1412 | "longitude":-72.5126,
1413 | "state":"MA"
1414 | },
1415 | {
1416 | "city":"Charleston",
1417 | "latitude":38.3498,
1418 | "longitude":-81.6326,
1419 | "state":"WV"
1420 | },
1421 | {
1422 | "city":"Yakima",
1423 | "latitude":46.6021,
1424 | "longitude":-120.5059,
1425 | "state":"WA"
1426 | },
1427 | {
1428 | "city":"CommerceCity",
1429 | "latitude":39.8083,
1430 | "longitude":-104.9339,
1431 | "state":"CO"
1432 | },
1433 | {
1434 | "city":"Hamtramck",
1435 | "latitude":42.3928,
1436 | "longitude":-83.0496,
1437 | "state":"MI"
1438 | },
1439 | {
1440 | "city":"Eureka",
1441 | "latitude":40.8021,
1442 | "longitude":-124.1637,
1443 | "state":"CA"
1444 | },
1445 | {
1446 | "city":"BentonHarbor",
1447 | "latitude":42.1167,
1448 | "longitude":-86.4542,
1449 | "state":"MI"
1450 | },
1451 | {
1452 | "city": "Montclair",
1453 | "latitude": 34.0775104,
1454 | "longitude": -117.6897776,
1455 | "state": "CA"
1456 | },
1457 | {
1458 | "city":"Greenfieldtown",
1459 | "latitude":42.5879,
1460 | "longitude":-72.5994,
1461 | "state":"MA"
1462 | }
1463 | ]
```