API for the Hopsworks Feature Store

Hopsworks has a data management layer for machine learning, called a feature store. The feature store enables simple and efficient versioning, sharing, governance and definition of features that can be used to both train machine learning models or to serve inference requests. The featurestore serves as a natural interface between data engineering and data science.

API documentation

Reading from the featurestore:

from hops import featurestore
features_df = featurestore.get_features(["team_budget", "average_attendance", "average_player_age"])

Integration with Sci-kit Learn:

from hops import featurestore
train_df = featurestore.get_featuregroup("iris_features", dataframe_type="pandas")
x_df = train_df[['sepal_length', 'sepal_width', 'petal_length', 'petal_width']]
y_df = train_df[["label"]]
X = x_df.values
y = y_df.values.ravel()
iris_knn = KNeighborsClassifier()
iris_knn.fit(X, y)

Integration with Tensorflow:

from hops import featurestore
feature_list = ["team_budget", "average_attendance", "average_player_age",
    "team_position", "sum_attendance",
    "average_player_rating", "average_player_worth", "sum_player_age",
    "sum_player_rating", "sum_player_worth", "sum_position",
    "average_position"
  ]

latest_version = featurestore.get_latest_training_dataset_version("team_position_prediction")
featurestore.create_training_dataset(
    features = feature_list,
    training_dataset = "team_position_prediction",
    descriptive_statistics = False,
    feature_correlation = False,
    feature_histograms = False,
    cluster_analysis = False,
    training_dataset_version = latest_version + 1
)

def create_tf_dataset():
    dataset_dir = featurestore.get_training_dataset_path("team_position_prediction")
    input_files = tf.gfile.Glob(dataset_dir + "/part-r-*")
    dataset = tf.data.TFRecordDataset(input_files)
    tf_record_schema = ... # Add tf schema
    feature_names = ["team_budget", "average_attendance", "average_player_age", "sum_attendance",
         "average_player_rating", "average_player_worth", "sum_player_age", "sum_player_rating", "sum_player_worth",
         "sum_position", "average_position"
        ]
    label_name = "team_position"

    def decode(example_proto):
        example = tf.parse_single_example(example_proto, tf_record_schema)
        x = []
        for feature_name in feature_names:
            x.append(example[feature_name])
        y = [tf.cast(example[label_name], tf.float32)]
        return x,y

    dataset = dataset.map(decode).shuffle(SHUFFLE_BUFFER_SIZE).batch(BATCH_SIZE).repeat(NUM_EPOCHS)
    return dataset

tf_dataset = create_tf_dataset()

Feature Visualizations: