compositional-qsvm/data_preprocessing.py at main · PauMayench/compositional-qsvm · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
import pandas as pd
from sklearn.model_selection import train_test_split

def data_preprocessing(dataset_name="perovskites"):
    """
    returns the numpy arrays as the follwoing format:
    train_X, train_Y, validation_X, validation_Y, test_X, test_Y

    X are arrays of the shape = [  [x1_1, x1_2, x1_3, x1_4],   [x2_1, x2_2, x2_3, x2_4] ... ]
    y are arrays of the shape = [ y1, y2, y3, y4 ]     <- they are values either 1 or -1


    COPY:

    X_train, y_train, X_val, y_val, X_test, y_test = get_dataset()

    """

    if dataset_name == "perovskites":


        df = pd.read_csv("datasets/halide_perovskites_4D_ionic_radii.csv")


        # add label field        metal -> 1        non  metal -> -1
        df.loc[df['is_metal'] == True, 'label'] = 1
        df.loc[df['is_metal'] == False, 'label'] = -1

        #choose only the rows that we want
        X =  df[["r_A", "r_B", "r_Bprime", "r_X"]]
        y =  df['label']

        train_X, rest_X, train_Y, rest_y = train_test_split(X, y, train_size=100, random_state=42) # we split it two, times this is the first one
        validation_X, test_X, validation_Y, test_Y = train_test_split(rest_X, rest_y, train_size=100, test_size=1442, random_state=42) # train_size is validation

        #the paper splits
        #   100 -> train
        #   100 -> validation
        #   1442 -> test

        return train_X.values, train_Y.values, validation_X.values, validation_Y.values, test_X.values, test_Y.values # .values converst pandas to np arrays


    if dataset_name == "ad-hoc":
        #the paper splits
        #   100 -> train
        #   100 -> validation
        #   4100 -> test


        data_file = "datasets/synthetic_adhoc_3D.csv"
        feature_cols = ["feature_1", "feature_2", "feature_3"]

        df = pd.read_csv(data_file)
        df.loc[df['target'] == 1, 'label'] = 1
        df.loc[df['target'] == 0, 'label'] = -1

        train_df = df[df["split"] == "train"]
        val_df = df[df["split"] == "validation"]
        test_df = df[df["split"] == "test"]

        train_X = train_df[feature_cols].values
        train_Y = train_df["label"].values
        validation_X = val_df[feature_cols].values
        validation_Y = val_df["label"].values
        test_X = test_df[feature_cols].values
        test_Y = test_df["label"].values

        return train_X, train_Y, validation_X, validation_Y, test_X, test_Y # .values converst pandas to np arrays

    if dataset_name == "primes":
        #the paper splits
        #   100 -> train
        #   100 -> validation
        #   3896 -> test


        data_file = "datasets/primes.csv"
        feature_cols = ["x2", "x1", "x0"]

        df = pd.read_csv(data_file)

        # add label field
        df.loc[df['target'] == 1, 'label'] = 1
        df.loc[df['target'] == 0, 'label'] = -1

        #choose only the rows that we want
        X =  df[["x2", "x1", "x0"]]
        y =  df['label']

        train_X, rest_X, train_Y, rest_y = train_test_split(X, y, train_size=100, random_state=42) # we split it two, times this is the first one
        validation_X, test_X, validation_Y, test_Y = train_test_split(rest_X, rest_y, train_size=100, test_size=3896, random_state=42) # train_size is validation


        return train_X.values, train_Y.values, validation_X.values, validation_Y.values, test_X.values, test_Y.values # .values converst pandas to np arrays

    if dataset_name == "hidden_manifold":
        #the paper splits
        #   100 -> train
        #   100 -> validation
        #   41 -> test

        data_file = "datasets/hidden_manifold.csv"
        feature_cols = ["x", "y", "z"]

        df = pd.read_csv(data_file)

        #choose only the rows that we want
        X =  df[["x", "y", "z"]]
        y =  df['label']

        train_X, rest_X, train_Y, rest_y = train_test_split(X, y, train_size=100, random_state=42) # we split it two, times this is the first one
        validation_X, test_X, validation_Y, test_Y = train_test_split(rest_X, rest_y, train_size=100, test_size=40, random_state=42) # train_size is validation

        return train_X.values, train_Y.values, validation_X.values, validation_Y.values, test_X.values, test_Y.values # .values converst pandas to np arrays

    raise Exception("dataset name not implemented")


if __name__ == "__main__":
    d = data_preprocessing("primes")
    print(d)