def _load_data():

dfTrain = pd.read_csv(config.TRAIN_FILE)

dfTest = pd.read_csv(config.TEST_FILE)

def preprocess(df):

cols = [c for c in df.columns if c not in ["id", "target"]]

df["missing_feat"] = np.sum((df[cols] == -1).values, axis=1)

df["ps_car_13_x_ps_reg_03"] = df["ps_car_13"] * df["ps_reg_03"]

return df

dfTrain = preprocess(dfTrain)

dfTest = preprocess(dfTest)

cols = [c for c in dfTrain.columns if c not in ["id", "target"]]

cols = [c for c in cols if (not c in config.IGNORE_COLS)]

X_train = dfTrain[cols].values

y_train = dfTrain["target"].values

X_test = dfTest[cols].values

ids_test = dfTest["id"].values

cat_features_indices = [i for i,c in enumerate(cols) if c in config.CATEGORICAL_COLS]

return dfTrain, dfTest, X_train, y_train, X_test, ids_test, cat_features_indices

def _run_base_model_dfm(dfTrain, dfTest, folds, dfm_params):

fd = FeatureDictionary(dfTrain=dfTrain, dfTest=dfTest,

numeric_cols=config.NUMERIC_COLS,

ignore_cols=config.IGNORE_COLS)

data_parser = DataParser(feat_dict=fd)

Xi_train, Xv_train, y_train = data_parser.parse(df=dfTrain, has_label=True)

Xi_test, Xv_test, ids_test = data_parser.parse(df=dfTest)

dfm_params["feature_size"] = fd.feat_dim

dfm_params["field_size"] = len(Xi_train[0])

y_train_meta = np.zeros((dfTrain.shape[0], 1), dtype=float)

y_test_meta = np.zeros((dfTest.shape[0], 1), dtype=float)

_get = lambda x, l: [x[i] for i in l]

gini_results_cv = np.zeros(len(folds), dtype=float)

gini_results_epoch_train = np.zeros((len(folds), dfm_params["epoch"]), dtype=float)

gini_results_epoch_valid = np.zeros((len(folds), dfm_params["epoch"]), dtype=float)

for i, (train_idx, valid_idx) in enumerate(folds):

Xi_train_, Xv_train_, y_train_ = _get(Xi_train, train_idx), _get(Xv_train, train_idx), _get(y_train, train_idx)

Xi_valid_, Xv_valid_, y_valid_ = _get(Xi_train, valid_idx), _get(Xv_train, valid_idx), _get(y_train, valid_idx)

dfm = DeepFM(**dfm_params)

dfm.fit(Xi_train_, Xv_train_, y_train_, Xi_valid_, Xv_valid_, y_valid_)

y_train_meta[valid_idx,0] = dfm.predict(Xi_valid_, Xv_valid_)

y_test_meta[:,0] += dfm.predict(Xi_test, Xv_test)

gini_results_cv[i] = gini_norm(y_valid_, y_train_meta[valid_idx])

gini_results_epoch_train[i] = dfm.train_result

gini_results_epoch_valid[i] = dfm.valid_result

y_test_meta /= float(len(folds))

# save result

if dfm_params["use_fm"] and dfm_params["use_deep"]:

clf_str = "DeepFM"

elif dfm_params["use_fm"]:

clf_str = "FM"

elif dfm_params["use_deep"]:

clf_str = "DNN"

print("%s: %.5f (%.5f)"%(clf_str, gini_results_cv.mean(), gini_results_cv.std()))

filename = "%s_Mean%.5f_Std%.5f.csv"%(clf_str, gini_results_cv.mean(), gini_results_cv.std())

_make_submission(ids_test, y_test_meta, filename)

_plot_fig(gini_results_epoch_train, gini_results_epoch_valid, clf_str)

return y_train_meta, y_test_meta

class DataParser(object):

def __init__(self, feat_dict):

self.feat_dict = feat_dict #這個feat_dict是FeatureDictionary對象實例

def parse(self, infile=None, df=None, has_label=False):

assert not ((infile is None) and (df is None)), "infile or df at least one is set"

assert not ((infile is not None) and (df is not None)), "only one can be set"

if infile is None:

dfi = df.copy()

else:

dfi = pd.read_csv(infile)

if has_label:

y = dfi["target"].values.tolist()

dfi.drop(["id", "target"], axis=1, inplace=True)

else:

ids = dfi["id"].values.tolist()

dfi.drop(["id"], axis=1, inplace=True)

# dfi for feature index

# dfv for feature value which can be either binary (1/0) or float (e.g., 10.24)

dfv = dfi.copy()

for col in dfi.columns:

if col in self.feat_dict.ignore_cols:

dfi.drop(col, axis=1, inplace=True)

dfv.drop(col, axis=1, inplace=True)

continue

if col in self.feat_dict.numeric_cols:

dfi[col] = self.feat_dict.feat_dict[col]

else:

dfi[col] = dfi[col].map(self.feat_dict.feat_dict[col])

dfv[col] = 1.

#dfi.to_csv('dfi.csv')

#dfv.to_csv('dfv.csv')

# list of list of feature indices of each sample in the dataset

Xi = dfi.values.tolist()

# list of list of feature values of each sample in the dataset

Xv = dfv.values.tolist()

if has_label:

return Xi, Xv, y

else:

return Xi, Xv, ids

def _init_graph(self):

self.graph = tf.Graph()

with self.graph.as_default():

tf.set_random_seed(self.random_seed)

self.feat_index = tf.placeholder(tf.int32, shape=[None, None],

name="feat_index") # None * F

self.feat_value = tf.placeholder(tf.float32, shape=[None, None],

name="feat_value") # None * F

self.label = tf.placeholder(tf.float32, shape=[None, 1], name="label") # None * 1

self.dropout_keep_fm = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_fm")

self.dropout_keep_deep = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_deep")

self.train_phase = tf.placeholder(tf.bool, name="train_phase")

self.weights = self._initialize_weights()

# model

self.embeddings = tf.nn.embedding_lookup(self.weights["feature_embeddings"],

self.feat_index) # None * F * K

#print(self.weights["feature_embeddings"]) shape=[259,8] n*k個隱向量

#print(self.embeddings) shape=[?,39,8] f*k 每個field取出一個隱向量[這不是FFM每個field取是在取非0量，減少計算]

feat_value = tf.reshape(self.feat_value, shape=[-1, self.field_size, 1])

#print(feat_value) shape=[?,39*1] 某一個樣本的39個Feature值

self.embeddings = tf.multiply(self.embeddings, feat_value) #multiply在有一個維度不同時，較少的維度會自行擴展

#print(self.embeddings) shape=[?,39*8]

# 所以這個multiply之后得到的矩陣是Vixi,方便以后進行<Vi,Vj>*xi*xj=<Vi*xi,Vj*xj>的計算，后面的計算FM被簡化為了

# sum_square part-square_sum part的形式,采用上面multiply的形式更方便啊！

# ---------- first order term ----------

self.y_first_order = tf.nn.embedding_lookup(self.weights["feature_bias"], self.feat_index) # None * F * 1

self.y_first_order = tf.reduce_sum(tf.multiply(self.y_first_order, feat_value), 2) # None * F

self.y_first_order = tf.nn.dropout(self.y_first_order, self.dropout_keep_fm[0]) # None * F

# ---------- second order term ---------------

# sum_square part

self.summed_features_emb = tf.reduce_sum(self.embeddings, 1) # None * K

self.summed_features_emb_square = tf.square(self.summed_features_emb) # None * K

# square_sum part

self.squared_features_emb = tf.square(self.embeddings)

self.squared_sum_features_emb = tf.reduce_sum(self.squared_features_emb, 1) # None * K

# second order

self.y_second_order = 0.5 * tf.subtract(self.summed_features_emb_square, self.squared_sum_features_emb) # None * K

self.y_second_order = tf.nn.dropout(self.y_second_order, self.dropout_keep_fm[1]) # None * K

# ---------- Deep component ----------

self.y_deep = tf.reshape(self.embeddings, shape=[-1, self.field_size * self.embedding_size]) # None * (F*K)

self.y_deep = tf.nn.dropout(self.y_deep, self.dropout_keep_deep[0])

for i in range(0, len(self.deep_layers)):

self.y_deep = tf.add(tf.matmul(self.y_deep, self.weights["layer_%d" %i]), self.weights["bias_%d"%i]) # None * layer[i] * 1

if self.batch_norm:

self.y_deep = self.batch_norm_layer(self.y_deep, train_phase=self.train_phase, scope_bn="bn_%d" %i) # None * layer[i] * 1

self.y_deep = self.deep_layers_activation(self.y_deep)

self.y_deep = tf.nn.dropout(self.y_deep, self.dropout_keep_deep[1+i]) # dropout at each Deep layer

# ---------- DeepFM ----------

if self.use_fm and self.use_deep:

concat_input = tf.concat([self.y_first_order, self.y_second_order, self.y_deep], axis=1)

elif self.use_fm:

concat_input = tf.concat([self.y_first_order, self.y_second_order], axis=1)

elif self.use_deep:

concat_input = self.y_deep

self.out = tf.add(tf.matmul(concat_input, self.weights["concat_projection"]), self.weights["concat_bias"])