np.array большое потребление памяти

sql = pd.read_sql(f'SELECT...', con=engine)
a = []
a.append(sql[i:i+window])

np.array(a, dtype=np.float32)

X_train = []
 
X_train.append(scaled[i-window:i])
y_train.append((0, 0, 0, 0, 0, 1))
 
print(type(X_train), type(X_train[0]), type(X_train[0][0]), type(X_train[0][0][0]))

X_train2 = np.array(X_train[:int((len(X_train) / 100) * 5)], dtype=np.float32)

X_train2 = pd.DataFrame(X_train)

X_train2 = np.array(X_train)

import numpy
import pandas
 
 
array = numpy.ndarray((1000, 500), numpy.float16)
dataframe = pandas.DataFrame(array, columns=range(1, 501))
dataframe.to_csv("example.csv", columns=range(1, 501), index=False)
 
 
with pandas.read_csv("example.csv", dtype=numpy.float16, chunksize=500) as dataframes:
    for dataframe in dataframes:
        array = dataframe.to_numpy()
        ...

np.savez_compressed("file", X_train2, y_train2)

file = np.load(f'file.npz')
 
X_train2 = file['arr_0']
y_train2 = file['arr_1']

import zarr
if not 'za_X_train' in globals():
    X_train2 = np.array(X_train, dtype=np.float32)
    y_train2 = np.array(y_train, dtype=np.int8)
    X_train2 = np.reshape(X_train2, (X_train2.shape[0], X_train2.shape[1], X_train2.shape[2]))
                                
    za_X_train = zarr.array(X_train2, chunks=(100, 256, 91), store='Xfile.zarr')
    za_y_train = zarr.array(y_train2, chunks=(100, 6), store='Yfile.zarr')
                                
    X_train = []
    y_train = []
                            
elif X_train != []:
    X_train2 = np.array(X_train, dtype=np.float32)
    y_train2 = np.array(y_train, dtype=np.int8)
    X_train2 = np.reshape(X_train2, (X_train2.shape[0], X_train2.shape[1], X_train2.shape[2]))
                                
    za_X_train.append(X_train2)
    za_y_train.append(y_train2)
                                
    X_train = []
    y_train = []

za_X_train.info
 
Type    zarr.core.Array
Data type   float32
Shape   (4991349, 256, 91)
Chunk shape (100, 256, 91)
Order   C
Read-only   False
Compressor  Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store type  zarr.storage.DirectoryStore
No. bytes   465113865216 (433.2G)
No. bytes stored    114050705890 (106.2G)
Storage ratio   4.1
Chunks initialized  49750/49914

from tensorflow.keras.utils import Sequence
 
class DataGenerator(Sequence):
 
  def __init__(self, X_zarr_file, y_zarr_file, output_size, shuffle=False, batch_size=10, ):
    """
    Initializes a data generator object
      :param csv_file: file in which image names and numeric labels are stored
      :param base_dir: the directory in which all images are stored
      :param output_size: image output size after preprocessing
      :param shuffle: shuffle the data after each epoch
      :param batch_size: The size of each batch returned by __getitem__
    """
    self.X_df = zarr.open(X_zarr_file, mode='r')
    self.y_df = zarr.open(y_zarr_file, mode='r')
    self.output_size = output_size
    self.shuffle = shuffle
    self.batch_size = batch_size
    self.on_epoch_end()
    
 
  def on_epoch_end(self):
    self.indices = np.arange(len(self.X_df))
    if self.shuffle:
      np.random.shuffle(self.indices)
 
  def __len__(self):
    return int(len(self.X_df) / self.batch_size)
 
  def __getitem__(self, idx):
    ## Initializing Batch
    #  that one in the shape is just for a one channel images
    # if you want to use colored images you might want to set that to 3
    X = np.empty((self.batch_size, *self.output_size, 1))
    # (x, y, h, w)
    y = np.empty((self.batch_size, 6, 1))
 
    # get the indices of the requested batch
    X = np.array(self.X_df[idx*self.batch_size:(idx+1)*self.batch_size])
    y = np.array(self.y_df[idx*self.batch_size:(idx+1)*self.batch_size])
 
    
 
    return X, y