PointManifold_dy/data_manifold_processing.py

#!/usr/bin/env python
# coding: utf-8

import os
import sys
import glob
import h5py
import numpy as np
from sklearn import manifold
from collections import OrderedDict
from functools import partial
from time import time


def load_data(partition):
    DATA_DIR = './data'
    all_data = []
    all_label = []
    for h5_name in glob.glob(os.path.join(DATA_DIR, 'modelnet40*hdf5_2048', '*%s*.h5'%partition)):
        f = h5py.File(h5_name, 'r+')
        data = f['data'][:].astype('float32')
        label = f['label'][:].astype('int64')
        f.close()
        all_data.append(data)
        all_label.append(label)
    all_data = np.concatenate(all_data, axis=0)
    all_label = np.concatenate(all_label, axis=0)
    return all_data, all_label

def load_data_by_cls(partition, class_label):
    DATA_DIR = './data'
    all_data = []
    all_label = []
    for h5_name in glob.glob(os.path.join(DATA_DIR, 'modelnet40*hdf5_2048', '*%s*.h5'%partition)):
        f = h5py.File(h5_name, 'r+')
        data = f['data'][:].astype('float32')
        label = f['label'][:].astype('int64')
        f.close()
        for i, item in enumerate(label):
            if item[0] == class_label:
                all_data.append(np.expand_dims(data[i], axis=0))
                all_label.append(np.expand_dims(label[i], axis=0))
    all_data = np.concatenate(all_data, axis=0)
    all_label = np.concatenate(all_label, axis=0)
    return all_data, all_label


def processing_data():

	train_data, train_label = load_data('train')
	test_data, test_label = load_data('test')

	n_neighbors=12
	n_components=2

	# Define methods
	LLE = partial(manifold.LocallyLinearEmbedding,
	              n_neighbors=n_neighbors, n_components=n_components, eigen_solver='auto')
	methods = OrderedDict()

	methods['LLE'] = LLE(method='standard')
	methods['LTSA'] = LLE(method='ltsa')
	methods['Hessian LLE'] = LLE(method='hessian')
	methods['Modified LLE'] = LLE(method='modified')
	methods['Isomap'] = manifold.Isomap(n_neighbors=n_neighbors, n_components=n_components)
	methods['MDS'] = manifold.MDS(n_components=n_components, max_iter=100, n_init=1)
	methods['SE'] = manifold.SpectralEmbedding(n_components=n_components,
	                                           n_neighbors=n_neighbors)
	methods['t-SNE'] = manifold.TSNE(n_components=n_components, init='pca',
	                                 random_state=0)
	# ### Isomap
	# train_data_h_lle = []
	# print('Hessian LLE processing...')
	# t0 = time()
	# for i,x in enumerate(train_data):
	# 	print('Processing sample%d'%i)
	# 	x_manifold = methods['Hessian LLE'].fit_transform(x)
	# 	x_new = np.concatenate((x, x_manifold),axis=1)
	# 	train_data_h_lle.append(x_new)
	# 	t1 = time()
	# print('Finish Hessian LLE!, use %.2g sec.'%(t1 - t0))

	# train_data_h_lle = np.array(train_data_h_lle)
	# f = h5py.File('./data/modelnet40_train_add_h_lle.h5','w')
	# f['data'] = train_data_h_lle
	# f['label'] = train_label
	# f.close()


	# # Process test data

	# test_data_h_lle = []
	# for i,x in enumerate(test_data):
	# 	print('Processing sample%d'%i)
	# 	x_manifold = methods['Hessian LLE'].fit_transform(x)
	# 	x_new = np.concatenate((x, x_manifold),axis=1)
	# 	test_data_h_lle.append(x_new)
	# print('Finish Hessian LLE!')
	# test_data_h_lle = np.array(test_data_h_lle)
	# f = h5py.File('./data/modelnet40_test_add_h_lle.h5','w')
	# f['data'] = test_data_h_lle
	# f['label'] = test_label
	# f.close()


	# train_data_m_lle = []
	# print('Modified LLE processing...')
	# t0 = time()
	# for i,x in enumerate(train_data):
	# 	print('Processing sample%d'%i)
	# 	x_manifold = methods['Modified LLE'].fit_transform(x)
	# 	x_new = np.concatenate((x, x_manifold),axis=1)
	# 	train_data_m_lle.append(x_new)
	# 	t1 = time()
	# print('Finish Modified LLE!, use %.2g sec.'%(t1 - t0))

	# train_data_m_lle = np.array(train_data_m_lle)
	# f = h5py.File('./data/modelnet40_train_add_m_lle.h5','w')
	# f['data'] = train_data_m_lle
	# f['label'] = train_label
	# f.close()


	# # Process test data

	# test_data_m_lle = []
	# for i,x in enumerate(test_data):
	# 	print('Processing sample%d'%i)
	# 	x_manifold = methods['Modified LLE'].fit_transform(x)
	# 	x_new = np.concatenate((x, x_manifold),axis=1)
	# 	test_data_m_lle.append(x_new)
	# print('Finish Modified LLE!')
	# test_data_m_lle = np.array(test_data_m_lle)
	# f = h5py.File('./data/modelnet40_test_add_m_lle.h5','w')
	# f['data'] = test_data_m_lle
	# f['label'] = test_label
	# f.close()

	# ### LTSA
	# train_data_ltsa = []
	# print('LTSA processing...')
	# t0 = time()
	# for i,x in enumerate(train_data):
	# 	print('Processing sample%d'%i)
	# 	x_manifold = methods['LTSA'].fit_transform(x)
	# 	x_new = np.concatenate((x, x_manifold),axis=1)
	# 	train_data_ltsa.append(x_new)
	# 	t1 = time()
	# print('Finish LTSA!, use %.2g sec.'%(t1 - t0))

	# train_data_ltsa = np.array(train_data_ltsa)
	# f = h5py.File('./data/modelnet40_train_add_ltsa.h5','w')
	# f['data'] = train_data_ltsa
	# f['label'] = train_label
	# f.close()


	# Process test data

	test_data_ltsa = []
	for i,x in enumerate(test_data):
		print('Processing sample%d'%i)
		x_manifold = methods['LTSA'].fit_transform(x)
		x_new = np.concatenate((x, x_manifold),axis=1)
		test_data_ltsa.append(x_new)
	print('Finish LTSA!')
	test_data_ltsa = np.array(test_data_ltsa)
	f = h5py.File('./data/modelnet40_test_add_ltsa.h5','w')
	f['data'] = test_data_ltsa
	f['label'] = test_label
	f.close()

if __name__ == "__main__":
	processing_data()