These are chat archives for rosshinkley/nightmare

23rd
May 2016
Hudson Atwell
@atwellpub
May 23 2016 03:06
I've made some good progress today. One thing I am stuck on is how to process more than one page. This code seems to shut the server down without an error message. Any ideas?
crawlPages: function( url ) {

    console.log(url);

    Actions.nightmare
        .goto( url )
        .wait('.container')                
        .evaluate( function() {        
            var ids = Array.from(document.getElementsByClassName('one')).map( (element)=>{ return element.href.substr(element.href.lastIndexOf('/') + 1); } )
            var info = Array.from(document.getElementsByClassName('two')).map( (element)=>{ return element.innerHTML; } )
            var next = document.querySelector(".next_page");
            if ( next ) {
                var next = document.querySelector(".next_page").getAttribute('href');
            } 

            return [ ids , info , next ]         
        })                
        .end()
        .then( function ( result ) {
            var ids = result[0];
            var info = result[1];                
            var next = result[2];                

            var id;
            for ( var i=0; i < ids.length; i++ ) {
                var user = new Object();
                var array = info[i].replace( / /g  , '' ).split("\n");

                user.id = parseInt(ids[i])
                user.info = array.filter(function(e){return e}); 

                Actions.queue[i] = user
            }

            if (next) {
                return Actions.crawlPages(next);                
            } else {
                console.log('done');
            } 

        })
        .catch(function(error) {
            console.error('Search failed:', error);
        })
},
Ross Hinkley
@rosshinkley
May 23 2016 15:59
@atwellpub you are calling end, which will end the instance
doesn't look like it gets recreated
Hudson Atwell
@atwellpub
May 23 2016 16:02
You are right.
:grin:
Ross Hinkley
@rosshinkley
May 23 2016 16:03
not sure that will solve all your problems, but it's certainly a start :)
Hudson Atwell
@atwellpub
May 23 2016 16:04
thank you
Ross Hinkley
@rosshinkley
May 23 2016 16:05
no problem