Issues with my Javascript code to parse files

62 Views Asked by At

I am currently working on implementing a JavaScript code that aims to read and parse some numpy .npy files generated from Python's numpy, specified by a JSON file. These .npy files contain arrays of floating values (a ML model weights and biases as arrays). However, I am encountering errors while running the script. What is causing the issues in my implementation? How to resolve them?

Here is the code I have written so far:

import fs from 'fs';
import path from 'path';
import numpy from 'numpy';

const jsonPath = 'model_weights.json';
const jsonData = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));

for (const [layerName, layerData] of Object.entries(jsonData)) {
  console.log(`Layer: ${layerName}`);

  // Read and print the weight data
  const weightFile = layerData.weight;
  const weightData = numpy.load(weightFile);
  console.log('Weight:');
  console.log(weightData);

  // Read and print the bias data
  const biasFile = layerData.bias;
  const biasData = numpy.load(biasFile);
  console.log('Bias:');
  console.log(biasData);

  console.log();
}

And here is a sample JSON file:

{
    "conv_mid.0": {
        "weight": "conv_mid.0_weight.npy",
        "bias": "conv_mid.0_bias.npy"
    },
    "conv_mid.1": {
        "weight": "conv_mid.1_weight.npy",
        "bias": "conv_mid.1_bias.npy"
    },
    "conv_mid.2": {
        "weight": "conv_mid.2_weight.npy",
        "bias": "conv_mid.2_bias.npy"
    },
    "conv_mid.3": {
        "weight": "conv_mid.3_weight.npy",
        "bias": "conv_mid.3_bias.npy"
    }
}

And here is the error I get:

node:internal/modules/esm/resolve:214
  const resolvedOption = FSLegacyMainResolve(packageJsonUrlString, packageConfig.main, baseStringified);
                         ^

Error: Cannot find package 'node_modules\numpy\package.json' imported from retrieve.mjs

For context, here is how I generate the .npy files:

import torch
import json
import numpy as np

state_dict = torch.load('network.pth')

# Extract the 'params' OrderedDict from the state_dict
params_dict = state_dict['params']

indices = {}

for name, param in params_dict.items():
    layer_name, param_type = name.rsplit('.', 1)

    if layer_name not in indices:
        indices[layer_name] = {}

    if param_type == 'weight':
        # Store the weight data as a numpy array
        data = np.array(param.tolist(), dtype=np.float32)
        np.save(f"{layer_name}_weight.npy", data)
        indices[layer_name]['weight'] = f"{layer_name}_weight.npy"
    elif param_type == 'bias':
        # Store the bias data as a numpy array
        data = np.array(param.tolist(), dtype=np.float32)
        np.save(f"{layer_name}_bias.npy", data)
        indices[layer_name]['bias'] = f"{layer_name}_bias.npy"

json_data = json.dumps(indices, indent=4)

with open('model_weights.json', 'w') as f:
    f.write(json_data)

print("JSON data and weights/biases saved to model_weights.json and .npy files")
print(json_data)

UPDATE: when I print loadedData this is the output:

{
  'conv_head.bias': [
    -0.00030437775421887636,
    0.0042824335396289825,
    0.0022352728992700577,
    -0.0019111455185338855,
    0.00017375686729792506,
    0.0017428217688575387,
    0.005364884156733751,
    0.0028239202219992876
  ],
  'conv_mid.1.bias': [
    -0.002030380303040147,
    0.009842530824244022,
    -0.0008345193928107619,
    -0.007043828722089529,
    -0.00633968273177743,
    -0.006188599392771721,
    -0.0018206692766398191,
    0.0037471423856914043
  ],
  'conv_head.weight': [
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ]
  ],
  'conv_mid.0.weight': [
    [
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array]
    ],
    [
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array]
    ], ...

And here is the python version to read the files which works fine:

import numpy as np
import json

# Load the JSON data containing the file names and indices
with open('model_weights.json', 'r') as f:
    json_data = json.load(f)

# Iterate over the layers
for layer_name, layer_data in json_data.items():
    print(f"Layer: {layer_name}")

    # Read and print the weight data
    weight_file = layer_data['weight']
    weight_data = np.load(weight_file)
    print("Weight:")
    print(weight_data)

    # Read and print the bias data
    bias_file = layer_data['bias']
    bias_data = np.load(bias_file)
    print("Bias:")
    print(bias_data)

    print()
0

There are 0 best solutions below