The following code is a project which parses directories of CSV files to input into a MySQL database. It often works, however I am plagued by an ETIMEDOUT error. I am a junior developer and am unsure how to handle this situation, or why it is occurring. It may be because I am parsing too much information at once. Any advise on how to handle this would be greatly appreciated.

const app = require('express')();
var fs = require('fs');
var Papa = require('papaparse');
var Sequelize = require('sequelize');
var _ = require('underscore');
var mainDirectory = ['./Relay-Data'];

var sequelize = new Sequelize('relay_cloud','root','admin',{
  host: 'localhost',
  dialect: 'mysql',
  logging: false,
    pool: {
     max: 5,
     min: 0,
     idle: 50000,
     acquire: 50000,
     evict: 50000,
     handleDisconnects: true
 }
});

var Relay = sequelize.define('relay',{
  Timestamp:{
    type: Sequelize.DATE,
    field: 'Timestamp',
  },
  Identifier:{
    type: Sequelize.INTEGER,
    field: 'Identifier',
  },
  Building:{
    type: Sequelize.STRING,
    field: 'Building',
  },
  Device_Type:{
    type: Sequelize.STRING,
    field: 'Device_Type'
  },
  Unit:{
    type: Sequelize.STRING,
    field: 'Unit'
  },
  Energy:{
    type: Sequelize.DECIMAL,
    field: 'Energy'
  },
  Volume:{
    type: Sequelize.DECIMAL,
    field: 'Volume'
  },
  File_Name:{
    type: Sequelize.STRING,
    field: 'File_Name'
  }
},{
  timestamps: false,
  freezeTableName: true
});

Promise.all(mainDirectory.map(function(main) {
  return new Promise(function(resolve, reject) {
        fs.readdir(main, function(er,dirs){
            if (er) {
                return reject(er);
            }
            for(var i = 0; i < dirs.length; i++){
                dirs[i] = mainDirectory + "/" + dirs[i];
            }
            resolve(dirs);
        });
  }).then(function(dirs){
        return Promise.all(dirs.map(function(dir) {
          return new Promise(function(resolve, reject) {
            fs.readdir(dir, function(er, files) {
              if (er) {
                return reject(er);
              }
              resolve(files);
            });
          }).then(function(files) {
            return Promise.all(files.map(function(file) {
              return new Promise(function(resolve, reject) {
                fs.readFile(dir + '/' + file, 'utf-8', function(er, content) {
                  if (er) {
                    return reject(er);
                  }
                            var data_obj = Papa.parse(content,{
                                    quotes: false,
                                    delimiter: ";",
                                });
                                data_array = data_obj.data;
                                output_data_array = [];

                                for(var i = 10; i < data_array.length; i++){
                                    if(data_array[i][0] !== "" ){
                                        output_data_array.push({
                                                "Timestamp"     : data_array[i][0],
                                                "Identifier"    : data_array[i][1],
                                                "Building"      : file.split( "_" )[ 1 ],
                                                "Device_Type" : data_array[i][3],
                                                "Unit"          : data_array[i][5],
                                                "Energy"        : parseFloat(data_array[i][11].replace(',','.').replace(' ', '')) || 0 ,
                                                "Volume"    : parseFloat(data_array[i][13].replace(',','.').replace(' ', '')) || 0,
                                                "File_Name" : file
                                        });
                                    }
                                }
                        Relay.bulkCreate(output_data_array);
                    });
                resolve();
              });
            }));
          });
        }));
    });
})).catch(function(er) {
  console.log(er);
});



app.listen(process.env.PORT || 3000, process.env.IP, function(){
    console.log("Server is listening");
})

The following code is a project which parses directories of CSV files to input into a MySQL database. It often works, however I am plagued by an ETIMEDOUT error. I am a junior developer and am unsure how to handle this situation, or why it is occurring. It may be because I am parsing too much information at once. Any advise on how to handle this would be greatly appreciated.

const app = require('express')();
var fs = require('fs');
var Papa = require('papaparse');
var Sequelize = require('sequelize');
var _ = require('underscore');
var mainDirectory = ['./Relay-Data'];

var sequelize = new Sequelize('relay_cloud','root','admin',{
  host: 'localhost',
  dialect: 'mysql',
  logging: false,
    pool: {
     max: 5,
     min: 0,
     idle: 50000,
     acquire: 50000,
     evict: 50000,
     handleDisconnects: true
 }
});

var Relay = sequelize.define('relay',{
  Timestamp:{
    type: Sequelize.DATE,
    field: 'Timestamp',
  },
  Identifier:{
    type: Sequelize.INTEGER,
    field: 'Identifier',
  },
  Building:{
    type: Sequelize.STRING,
    field: 'Building',
  },
  Device_Type:{
    type: Sequelize.STRING,
    field: 'Device_Type'
  },
  Unit:{
    type: Sequelize.STRING,
    field: 'Unit'
  },
  Energy:{
    type: Sequelize.DECIMAL,
    field: 'Energy'
  },
  Volume:{
    type: Sequelize.DECIMAL,
    field: 'Volume'
  },
  File_Name:{
    type: Sequelize.STRING,
    field: 'File_Name'
  }
},{
  timestamps: false,
  freezeTableName: true
});

Promise.all(mainDirectory.map(function(main) {
  return new Promise(function(resolve, reject) {
        fs.readdir(main, function(er,dirs){
            if (er) {
                return reject(er);
            }
            for(var i = 0; i < dirs.length; i++){
                dirs[i] = mainDirectory + "/" + dirs[i];
            }
            resolve(dirs);
        });
  }).then(function(dirs){
        return Promise.all(dirs.map(function(dir) {
          return new Promise(function(resolve, reject) {
            fs.readdir(dir, function(er, files) {
              if (er) {
                return reject(er);
              }
              resolve(files);
            });
          }).then(function(files) {
            return Promise.all(files.map(function(file) {
              return new Promise(function(resolve, reject) {
                fs.readFile(dir + '/' + file, 'utf-8', function(er, content) {
                  if (er) {
                    return reject(er);
                  }
                            var data_obj = Papa.parse(content,{
                                    quotes: false,
                                    delimiter: ";",
                                });
                                data_array = data_obj.data;
                                output_data_array = [];

                                for(var i = 10; i < data_array.length; i++){
                                    if(data_array[i][0] !== "" ){
                                        output_data_array.push({
                                                "Timestamp"     : data_array[i][0],
                                                "Identifier"    : data_array[i][1],
                                                "Building"      : file.split( "_" )[ 1 ],
                                                "Device_Type" : data_array[i][3],
                                                "Unit"          : data_array[i][5],
                                                "Energy"        : parseFloat(data_array[i][11].replace(',','.').replace(' ', '')) || 0 ,
                                                "Volume"    : parseFloat(data_array[i][13].replace(',','.').replace(' ', '')) || 0,
                                                "File_Name" : file
                                        });
                                    }
                                }
                        Relay.bulkCreate(output_data_array);
                    });
                resolve();
              });
            }));
          });
        }));
    });
})).catch(function(er) {
  console.log(er);
});



app.listen(process.env.PORT || 3000, process.env.IP, function(){
    console.log("Server is listening");
})

• javascript
• mysql
• node.js
• sequelize.js
• papaparse

Share Improve this question Follow asked Dec 14, 2017 at 14:27 andar1anandar1an 12122 silver badges1212 bronze badges 2

• it looks like something in your query takes up too much time, causing the server to kill the connection. I would debug this with console.time and see where the bottleneck is. – asosnovsky Commented Dec 14, 2017 at 14:29
• 1 Thank you @asosnovsky! I was able to determine that it was a conflict between bulkCreate running too close to subsequent instances. – andar1an Commented Dec 14, 2017 at 20:03

Add a ment  | 

2 Answers 2

Sorted by: Reset to default 2

I was able to prevent the ETIMEDOUT error by doing the following:

setTimeout(function() {
            Relay.bulkCreate(output_data_array);
          }, 2);

I've been facing this issue today, and this worked for me:

let sequelize = new Sequelize(dbName, dbUsername, dbPassword, {
    host: dbHost,
    // the rest of your sequelize configuration...
    retry: {
        match: [
            /ETIMEDOUT/,
            /EHOSTUNREACH/,
            /ECONNRESET/,
            /ECONNREFUSED/,
            /ETIMEDOUT/,
            /ESOCKETTIMEDOUT/,
            /EHOSTUNREACH/,
            /EPIPE/,
            /EAI_AGAIN/,
            /SequelizeConnectionError/,
            /SequelizeConnectionRefusedError/,
            /SequelizeHostNotFoundError/,
            /SequelizeHostNotReachableError/,
            /SequelizeInvalidConnectionError/,
            /SequelizeConnectionTimedOutError/
        ],
        max: 5
    }
});