nodejs读取大文件测试

Posted by yuk on 2016-04-06

我就是想试试 lei-stream…..这个读写大文件
最近要一堆游戏log日志,逐行读取,文件几百兆.不知道v8引擎性能怎么样.

1
npm install lei-stream --save

文件大小： 368.1MB 行数25000+行

一，按块读取,控制读取速度

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

var util = require('util');
var events = require('events');
var fs = require('fs');

// 一个几百M的文本文件
var inputFile = './logs/money_change_2016-03-30.log';

function ReadStreamThrottle (stream, speed) {
    this._stream = stream;
    this._readBytes = 0;
    this._speed = speed;
    this._ended = false;
    this._readBytesSecond = 0;
    this._lastTimestamp = Date.now();
    this._paused = false;
    var self = this;

    // 检查速度是否太快
    function isTooFast () {
        var t = (Date.now() - self._lastTimestamp) / 1000;
        var bps = self._readBytesSecond / t;
        return bps > speed;
    }

    // 每隔一段时间检查速度
    function checkSpeed () {
        if (isTooFast()) {
            self.pause();
            // 直到平均速度放缓到预设的值时继续读流
            var tid = setInterval(function () {
                if (!isTooFast()) {
                    clearInterval(tid);
                    self.resume();
                }
            }, 100);
        } else {
            self.resume();
        }
    }

    stream.on('data', function (chunk) {
        self._readBytes += chunk.length;
        self._readBytesSecond += chunk.length;
        self.emit('data', chunk);
        checkSpeed();
    });

    stream.on('end', function () {
        self._ended = true;
        self.emit('end');
    });
}

util.inherits(ReadStreamThrottle, events.EventEmitter);

ReadStreamThrottle.prototype.pause = function () {
    this._paused = true;
    this._stream.pause();
};

ReadStreamThrottle.prototype.resume = function () {
    this._paused = false;
    this._stream.resume();
};

function mb (v) {
    return (v / 1024 / 1024).toFixed(2) + 'MB';
}

// 读取文件，限制速度不大于10MB/S
var MB = 1024 * 1024;
var s = new ReadStreamThrottle(fs.createReadStream(inputFile), MB * 500);
var bytes = 0;
var t = Date.now();
s.on('data', function (c) {
    bytes += c.length;
    var spent = (Date.now() - t) ;
console.log('read %s bytes, speed: %sMB/s, spend:%sms',mb( bytes ), (bytes / MB / ( spent / 1000) ).toFixed(2), spent.toFixed(2));

});
s.on('end', function () {
    var spent = (Date.now() - t);
    console.log('end. total %s bytes, %sms', bytes, spent);
    console.log(tt);
});

结果

1
2
3
4
5
6
...
read 350.88MB bytes, speed: 81.73MB/s, spend:4293.00ms
read 350.94MB bytes, speed: 81.73MB/s, spend:4294.00ms
read 351.00MB bytes, speed: 81.74MB/s, spend:4294.00ms
read 351.03MB bytes, speed: 81.75MB/s, spend:4294.00ms
end. total 368078169 bytes, 4296ms

其实输出占用大量了时间,注释掉之后实际用时 465ms…我的天
但是这种方法是读取块，无法按行筛选结果,所以忽略。。。。

二，按行读取

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
console.log('================app start!===============');
var readLine = require('lei-stream').readLine;

// 一个几百M的文本文件
var inputFile = './logs/money_change_2016-03-30.log';


var counter = 0;
var startTime = Date.now();

function msToS (v) {
    return parseInt(v / 1000, 10);
}

function getSpentTime () {
    return Date.now() - startTime;
}


var result = [];
readLine(inputFile).go(function (data, next) {
    counter++;
    //这里只是用来测试查找结果中有『Peterson Eileen, 』和『 ,1,』的结果并输出
    if( data.indexOf('Peterson Eileen,')>=0 && data.indexOf(',1,')>=0 ){
        result.push(data);
    }
    if (counter % 100000 === 0) {
        printSpeedInfo();
    }
    next();
}, function () {
    console.log('result length :%s', result.length)
    console.log('done. total %s lines, spent %sMS', counter, getSpentTime());
    printMemoryUsage();
    process.exit();
});



// 打印进度
function printSpeedInfo () {
    var t = getSpentTime();
    var s = counter / msToS(t);
    if (!isFinite(s)) s = counter;
    console.log('read %s lines, speed: %sL/S, spent: %sms', counter, s.toFixed(0), t.toFixed(0));
}

// 打印内存占用情况
function printMemoryUsage () {
    var info = process.memoryUsage();
    function mb (v) {
        return (v / 1024 / 1024).toFixed(2) + 'MB';
    }
    console.log('rss=%s, heapTotal=%s, heapUsed=%s', mb(info.rss), mb(info.heapTotal), mb(info.heapUsed));
}
setInterval(printMemoryUsage, 1000);
[/code]
[code lang="bash"]
================app start!===============
rss=69.02MB, heapTotal=48.11MB, heapUsed=12.54MB
rss=72.51MB, heapTotal=50.08MB, heapUsed=17.19MB
rss=72.58MB, heapTotal=50.08MB, heapUsed=16.55MB
result length :804
done. total 2529786 lines, spent 3307MS
rss=72.59MB, heapTotal=50.08MB, heapUsed=19.78MB

三 ，调用系统命令grep

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
var exec = require('child_process').exec;
var cmdStr ='more ./logs/money_change_2016-03-30.log | grep "Peterson Eileen," | grep ",1,"';

var startTime = Date.now();
exec(cmdStr, function(err,stdout,stderr){
    if(err) {
        console.log('get weather api error:'+stderr);
    } else {

        var data = stdout.split('n');
        var t = Date.now() - startTime;
        console.log( '%s ms',  t);
        console.log(data.length);
    }
});

结果稍慢一点.

1
2
5292 ms
805
FEATURED TAGS
FRIENDS