forked from: forked from: forked from: forked from: forked from: forked from: getToken

/**
 * Copyright tepe ( http://wonderfl.net/user/tepe )
 * MIT License ( http://www.opensource.org/licenses/mit-license.php )
 * Downloaded from: http://wonderfl.net/c/ghIw
 */

// forked from tepe's forked from: forked from: forked from: forked from: forked from: getToken
// forked from tepe's forked from: forked from: forked from: forked from: getToken
package {
    import flash.media.Video;
    //import flash.utils.Dictionary;
    import flash.display.*;
    import flash.text.*;
    import flash.events.*;
    import flash.net.*;
    import flash.utils.*;
    import net.hires.debug.Stats;
    


    public class FlashTest extends Sprite {
        private var t1:TextField = new TextField();//原文
        private var t2:TextField = new TextField();
        private var so : SharedObject;
        public function FlashTest() {
            
            // write as3 code here..
            init();
            //FPS計測    
            addChild( new Stats() );
            //Wonderfl.capture(stage);
            onChange(null);
            
        }
        
 
        private function init():void{
            //原文テキスト入力ボックス
            t1.border = true;
            t1.width = 230;
            t1.height = 360;
            t1.y=100;
            t1.type = "input";
            t1.multiline = true;
            so = SharedObject.getLocal("data");
            if(so){ 
                if(so.data.text != null){
                    t1.text = so.data.text;
                }
                else{    
                    t1.text "";
                    so.data.text = t1.text;
                }
            }
            addChild(t1);
            t1.addEventListener(Event.CHANGE,onChange);
            t2.type ="input";
            t2.x = 230;
            t2.width=230;
            t2.height=460;
            t2.wordWrap =true;
            t2.multiline=true;
            t2.border = true;
            addChild(t2);            
            
        }
        
        

        private var tk:Tokenizer;
        private function onChange(e:Event=null):void{
            
            
            tk = new Tokenizer();
            tk.addEventListener(Event.COMPLETE,onComplete);
            //進捗率の表示
            tk.addEventListener(ProgressEvent.PROGRESS,function(e:ProgressEvent):void{
                t2.text = Math.round(100*e.bytesLoaded/e.bytesTotal).toString()+"%";
            });

            tk.createToken(t1.text);
            //var cnt:int=0;
            t2.text = "";
        }
        
        //ステップリストとスコープリストの生成
        private function onComplete(e:Event):void{
            t2.text = "onComplete";

            var bStack:Vector.<Block> = new Vector.<Block>;
            var parent:Block;
            var bList:Vector.<Block> = new Vector.<Block>;
            var b:Block;
            var start:Token = tk.getToken();
            var startStack:Vector.<Token> = new Vector.<Token>;
            
            
            var cToken:Token = tk.getToken();

            while(cToken!=null){//無限ループ注意
                //記号の処理
                if(cToken.type == Token.MARK){
                    var s:String = cToken.text;
                    if(s == ";"){ 
                        b = new Block();
                        b.setArea(start,cToken);
                        b.parent = parent;
                        if(parent!=null){ 
                            parent.add(b);
                        }
                        
                        start = cToken.next;
                        bList.push(b);
                       
                        
                    }
                    else if(s=="{"){ //スコープ開始 
                        b = new Block();
                        b.setArea(start);
                        b.parent = parent;
                        
                        if(parent!=null){ 
                            parent.add(b);
                        }
                        parent = b;
                        bStack.push(b); 
                        
                        start = cToken.next;
                    }
                    else if(s=="}"){//スコープ終了
                        
                        b = bStack.pop();
                        b.setArea(b.start,cToken);
                        parent = b.parent;
                        bList.push(b);
                        start = cToken.next;
                    }
                    else if(s=="("){
                        startStack.push(start);
                        start = cToken.next;
                    }
                    else if(s== ")"){ 
                        ///if(start.prev.text=="(")
                        start = startStack.pop();
                    }
                    
                }
                cToken = cToken.next;
                //cnt++;
            }
            
            
            var i:int;
            var bufStr:String = new String();
            //bufStr += str;
            //t2.text = str;
            //return;

            for(i=0;i<bList.length;i++){ 
                var a:Block = bList[i].parent;
                var ar:Array = new Array();
                while(a!=null){//ツリーの表示
                    //bufStr+=a.id+"/";
                    if(a.head != null)ar.push(a.name);
                    else ar.push(a.id);
                    a=a.parent;
                }
                for(var k:int=1;k<=ar.length;k++){
                    bufStr+=""+ar[ar.length-k]+"\n";
                }

                bufStr+="\n";

                if(bList[i].parent==null)continue;
                if(bList[i].head == null){ 
                    bufStr+= bList[i].body+"\n-----------------\n";
                }
                else{
                    bufStr+=bList[i].head+"{ ... }\n";
                    if(bList[i].list!=null){
                    for(var j:int=0;j<bList[i].list.length;j++){
                        bufStr+=bList[i].list[j].id+" "; 
                    }
                    }
                    bufStr+="\n------------------\n";
                }

            }

            t2.text = bufStr;
            so.data.text = t1.text;
            
        }//function
    }//class
}//package
import flash.events.ProgressEvent;



//トークンオブジェクト
class Token{
    private static var idCnt:int;
    private var tokenId:int;
    private var _text:String;//文字列
    public var type:int;//タイプ
    //public var lineNum:int;//属するプロセスライン
    
    //トークンの並びを保持する
    public var next:Token;
    public var prev:Token;
    public static const ERROR:int = -1;//エラー
    public static const SPACE:int = 0;//スペース
    public static const COMMENT:int = 1;//コメント 
    public static const NUMBER:int = 2;//数値
    public static const WORD:int = 3;//識別子
    public static const TEXT:int = 4;//文字列
    public static const MARK:int = 5;//記号
    public static const OTHER:int = 6;//その他（マルチバイトの文字列)
    public static const SCOPE:int = 7;//スコープの置き換え
    public static const EOF:int = 8;//EOF
    
    public function Token(){
        idCnt++;
        tokenId = idCnt;
    }
    public function get id():int{
        return tokenId;
    }
    public function set text(str:String):void{
        _text = new String();
        _text += str;
        
    }
    public function get text():String{
        return _text;
    }

}

//トークンの移動操作をする
class TokenPoint{
    private var _current:Token;
    private var _start:Token;
    private var _end:Token;
    public function TokenPoint(t:Token,start:Token=null,end:Token=null){
        _current = t;
        _start = start;
        _end = end;
        
    }
    public function get getPos():Token{
        return _current;
    }

    public function movNext():int{//移動量を返す
        if(_current.next==null)return 0;
        _current = _current.next;
        return 1;
    }
    public function movPrev():int{
        if(_current.prev==null)return 0;
        _current = _current.prev;
        return -1;
    }
    //指定されたタイプの位置まで移動
    public function movNextType(type:int,end:Token=null):int{ 
        var cnt:int=0;
        var t:Token = _current;
        while(t.type!=type){ 
            if(t==null)return 0;
            if(t==end)return 0;
            t=t.next;
            cnt++;
        }
        _current = t;
        return cnt;
    }
    
    public function movPrevType(type:int,end:Token=null):int{
        var cnt:int=0;
        var t:Token = _current;
        while(t.type!=type){
            if(t==null)return 0;
            if(t==end)return 0;
            t=t.prev;
            cnt--;
        }
        _current = t;
        return cnt;

    }
    //指定された文字列の位置まで移動
    public function movNextText(text:String,end:Token=null):int{
        var cnt:int=0;
        var t:Token = _current;
        while(t.text != text){
            if(t==null)return 0;
            if(t==end)return 0;
            t=t.next;
            cnt++;
        }
        _current = t;
        return cnt;

    }
    
    public function movPrevText(text:String,end:Token=null):int{
        var cnt:int=0;
        var t:Token = _current;
        while(t.text != text){
            if(t==null)return 0;
            if(t==end)return 0;
            t=t.prev;
            cnt--;
        }
        _current = t;
        return cnt;
    }
}


class Block{ 
    private static var idCnt:int;
    private var _id:int;
    private var _name:String;
    private var _text:String;
    private var _head:String;
    private var _body:String;
    public var list:Vector.<Block>;
    
    private var _next:Block;
    private var _prev:Block;
    private var _parent:Block;
    private var _child:Block;
    
    private var _startToken:Token;
    private var _endToken:Token;
    private var _entryToken:Token;
    public function Block(){
        idCnt++;
        _id=idCnt;
    }
    //ID
    public function get id():int{
        return _id;
    }
    //要素数
    public function get length():int{
        if(list==null)return 0;
        return list.length;
    }
    public function add(block:Block):void{
        if(list == null)list = new Vector.<Block>;
        list.push(block);
        var len:int = list.length;
        if(1<len){ 
            list[len-1].prev = list[len-2];
            list[len-2].next = list[len-1];
        }
        _child = list[0];
        
    }
    
    public function get start():Token{
        return _startToken;
    }
    public function get end():Token{
        return _endToken;
    }


    public function get next():Block{//次の要素
        return _next;
    }
    public function get prev():Block{//前の要素
        return _prev;
    }
    public function get parent():Block{//上の階層
        return _parent;
    }
    public function get child():Block{//下の階層の先頭
        return _child;
    }
    
    public function set next(b:Block):void{ 
        if(_next == b)return;
        _next=b;
        if(_next==null)return;
        if(_next.prev.id!=this.id)_next.prev = this;
    }
    public function set prev(b:Block):void{ 
        _prev=b;
    }
    public function set parent(b:Block):void{
        _parent=b;
    }
    public function set child(b:Block):void{
        _child=b;
    }
    
    //トークン列の中から開始要素と終了要素を指定
    public function setArea(start:Token,end:Token=null):int{ 
        _text=null;
        _head=null;
        _body=null;
        _startToken = start;
        _endToken = end;
        if(end==null)return 3;//未完了
        if(_startToken.prev==null){//ルート
            if(_endToken.next==null)return 0;
        }

        //スコープの有無を確認
        if(_endToken.text ==";")return 1;
        if(_endToken.text =="}"){ 
            var t:Token = _startToken;
            var tp:TokenPoint = new TokenPoint(_startToken);
            if(tp.movNextText("{")==0)return -1;
            _entryToken = tp.getPos;
            return 2;
        }
        return -1;

    }
//-------------------------------------------------------------
    //テキスト取得
    public function get text():String{ 
        if(_text!=null)return _text;
        _text = new String();
        var t:Token = _startToken;
        while(t.next != null){
            _text += t.text;
            if(t.id == _endToken.id)break;
            t = t.next;
        }
        return _text;
    }
    public function get name():String{
        if(_name!=null)return _name;
        if(_entryToken==null)return null;
        
        var t:Token = _entryToken;
        var st:Token = _startToken;
        var cnt:int=0;
        //func(  [)]  :***{　このカッコの位置まで移動
        var tp:TokenPoint = new TokenPoint(_startToken);
        if(tp.movNextText("(")!=0){
            tp.movPrevType(Token.WORD);
        }

        t = _startToken;
        while(t!=null){ 
            if(t.text=="(")break;
            if(t.id==_entryToken.id)break;
            t = t.next;
            cnt++;
        }
        for(var i:int=0;i<cnt;i++){
            t=t.prev;
            if(t.type==Token.WORD)break;
        }
        
        _name=t.text;
        return _name;

    }

    
    public function get head():String{
        if(_entryToken==null)return null;
        if(_head!=null)return _head;
        _head = new String();
        var t:Token = _startToken;
        while(t.type==Token.SPACE || t.type==Token.COMMENT)t = t.next;//スペース、コメント除外
        while(t.next != null){
            _head += t.text;
            if(t.next.id == _entryToken.id)break;
            t = t.next;
        }
        return _head;

    }
    public function get body():String{ 
        if(_body!=null)return _body;
        _body = new String();
        var t:Token;
        
        if(_entryToken==null){ 
            t = _startToken;
            while(t.type==Token.SPACE || t.type==Token.COMMENT)t = t.next;//スペース、コメント除外
            while(t.next != null){
                _body += t.text;
                if(t.id == _endToken.id)break;
                t = t.next;
            }
            return _body;
        }
        t = _entryToken;
        while(t.next != null){
            _body += t.text;
            if(t.id == _endToken.id)break;
            t = t.next;
        }
        return _body;
    }

    

}




//Thread==========================================================================
import flash.utils.*;
import flash.events.*;
    
class Thread extends Timer{ 
    static public var SPAN:Number = 1000/30; 
    static public var RATE:Number = 0.5;
    public var end:Boolean = false, currentLoop:Loop = null;
    
    static private var NUM:int = 0;
    private var _limit:Number = 5, _time:int = 0, _added:Boolean, currentLoops:Vector.<Loop>, loops:Vector.<Loop> = new Vector.<Loop>();
    
    function Thread(){
        super(SPAN); start(); NUM++;
        addEventListener( "timer", onFrame  );
    }    
    public function loop( func:Function, onComplete:Function = null):void{
        var loop:Loop = new Loop( func, currentLoop );     
        if( currentLoop == null ){ loops.push( loop );
        }else{ currentLoop.loops.push( loop ); _added = true }
        if( onComplete != null ){
            loop = new Loop( onComplete, currentLoop );   
            if( currentLoop == null ){ loops.push( loop );
            }else{ currentLoop.loops.push( loop ); }
        }
    }    
    public function remove():void{
        NUM--; stop(); end = true;
        removeEventListener( "timer", onFrame  );
    }    
    private function onFrame(e:Event):void{
        _time = getTimer(); _limit = (SPAN * RATE) / NUM;
        all: while(true){
            if( currentLoop == null ){
                currentLoops = loops;
                if( loops.length == 0 ){ remove(); break; }
                while( currentLoops[0].loops.length != 0 ){ currentLoops = currentLoops[0].loops;  }
                currentLoop = currentLoops[0];
            }
            do{
                if( _limit < (getTimer() - _time)){ break all; }
                if( _added ){ _added = false; currentLoop = null; continue all;  }
            }while( currentLoop.func() )
            currentLoop = null;
            currentLoops.reverse(); currentLoops.pop(); currentLoops.reverse();
        }
    }
}
class Loop extends Object{
    public var func:Function, name:String, parent:Loop, loops:Vector.<Loop> = new Vector.<Loop>();
    function Loop( func:Function, parent:Loop = null, name:String = "" ){  
        this.func = func; this.parent = parent;
    }
}
//=======================================================================================



//////////////////////////////////////////////////////////////////////////
//  トークン列取得
//////////////////////////////////////////////////////////////////////////



//テキストからトークンを抽出する
class Tokenizer extends Thread{
    private var blockList:Vector.<Block>;
    private var tokenList:Vector.<Token>;
    //private var scopeList:Array;
    private var _length:int;
    private var source:String;
    private var cansel:Boolean;
    public function Tokenizer(){
        
    }
    //スコープリストを生成する
    public function createScopeList():int{
        return 0;
    }

    public function getEof():Token{
        return getToken(length-1);
    }


    //指定インデクスからmax個までトークン抽出
    //抽出したトークンはlistに追加
    //抽出したところまでのインデクスを返す
    public function createToken(str:String):void{
        
        source = str;
        tokenList = new Vector.<Token>;
        index01 = 0;
         
        var i:int = 0;
        var cnt:int = 0;
        loop( 
            function _while():Boolean{ 
                i= func01();
                
                if(cnt==0){
                    var e:ProgressEvent = new ProgressEvent(ProgressEvent.PROGRESS,false,true,i,str.length);
                    dispatchEvent(e);
                }
                cnt++;
                cnt%=500;

                return i<str.length;
            }, 
            function onComplete():void { 
                addToken("",Token.EOF);//EOF追加
                _length = tokenList.length;
                var event:Event = new Event(Event.COMPLETE);
                dispatchEvent(event);
                
            } 
        );
    }
    public function get length():int{
        return _length;
    }
    //先頭からの順を指定してトークンを取得
    public function getToken(index:int=0):Token{
        if(index<0 && tokenList.length<=index)return null;
        return tokenList[index];
    }
    
    //トークン列に追加
    private function addToken(str:String,type:int):int{
        var obj:Token = new Token();
        obj.text = str;
        obj.type = type;
        obj.next = null;
        tokenList.push(obj);
        if(1<tokenList.length){ 
            tokenList[tokenList.length-1].prev = tokenList[tokenList.length-2];
            tokenList[tokenList.length-2].next = tokenList[tokenList.length-1];
        }
        else tokenList[tokenList.length-1].prev = null;
        return tokenList.length;
    }



//-------------------------------------------------------------
    //開始インデクスを受け取りトークンのタイプを返す
    private var index01:int=0;
    private function func01():int{//
        if(index01 >= source.length)return source.length;
        var index:int = index01;
        var type:int;//トークンタイプ
        var str:String;//トークン文字列
        var reg:RegExp = new RegExp();                
        var result:Object;
        //トークンタイプ：空白か？
        if(    source.charAt(index)== " " ||
               source.charAt(index)=="\t" ||
               source.charAt(index)=="\n" ||
               source.charAt(index)=="\r" ){
                   type = Token.SPACE;
                   reg = /[^\s]/mg;
                   reg.lastIndex = index;
                   result = reg.exec(source);
                   //トークン作成
                   if(result == null){//
                       str = source.substring(index,source.length);
                       index01 = source.length;//次の開始位置
                   }
                   else{//
                       str = source.substring(index,result.index);
                       index01 = result.index;                       
                   }
                   addToken(str,type);
                   return index01;
                                       
               }

        
        //トークンタイプ:文字列定数１
        if(source.charAt(index)== "'"){
            type = Token.TEXT;
            var indexA:int = index;
            reg = /['\n\r]/mg;//改行か次の[']が現れる位置
            do{//エスケープ処理
                reg.lastIndex = indexA+1;
                result = reg.exec(source);
                if(result==null)break;
                else indexA = result.index;
            }while(source.charAt(indexA-1)=="\\");//["]が現れてもその前がエスケープなら再検索
            
            //トークン作成
            if(result == null){//エラー
                str = "'";
                type = Token.ERROR;
                index01 = index+1;//次の開始位置
            }
            else{//
                str = source.substring(index,result.index+1);
                index01 = result.index+1;//次の開始位置
            }
            addToken(str,type);
            return index01;
                
        }
 
        //トークンタイプ:文字列定数2
        if(source.charAt(index)== '"'){
            type = Token.TEXT;
            var indexB:int = index;
            reg = /["\n\r]/mg;//改行か次の["]が現れる位置
            do{//エスケープ処理
                reg.lastIndex = indexB+1;
                result = reg.exec(source);
                if(result==null)break;
                else indexB = result.index;
            }while(source.charAt(indexB-1)=="\\");//["]が現れてもその前がエスケープなら再検索
            
            //トークン作成
            if(result == null){//エラー
                str = '"';
                type = Token.ERROR;
                index01 = index+1;//次の開始位置
                
            }
            else{//
                str = source.substring(index,result.index+1);
                index01 = result.index+1;//次の開始位置
                
            }
            addToken(str,type);
            return index01;
        }
        
        if(source.charAt(index)== "/"){
               //一行コメント
               if(source.charAt(index+1)== "/"){//
                   reg = /[\n\r]/mg;
                   reg.lastIndex = index+2;
                   result = reg.exec(source);
                   //トークン作成
                   type = Token.COMMENT;
                   if(result == null){
                       str = source.substring(index,source.length);
                       index01 = source.length;//次の開始位置                       
                   }
                   else{// 
                       str = source.substring(index,result.index);
                       index01 = result.index;//次の開始位置
                   }
                   addToken(str,type);
                   return index01;
               }
               //複数行コメント
               if(source.charAt(index+1)== "*"){
                   reg = /\*\//mg;
                   reg.lastIndex = index+2;
                   result = reg.exec(source);
                   
                   //トークン作成
                   type = Token.COMMENT;
                   if(result == null){//エラー
                       type = Token.ERROR;
                       str = '/*';
                       index01 = index+2;//次の開始位置
                   }
                   else{//
                       str = source.substring(index,result.index+2);
                       index01 = result.index+2;//次の開始位置                       
                   }
                   addToken(str,type);
                   return index01;
               }
        }

        //英数字
        if((source.charCodeAt(index) > 64 && source.charCodeAt(index) < 91) ||// A-Z
           (source.charCodeAt(index) > 96 && source.charCodeAt(index) < 123)||// a-z
            source.charCodeAt(index) == 95 ){// _
                reg = /[^a-zA-Z0-9_]/mg;//英数字以外が現れる位置
                reg.lastIndex = index+1;
                result = reg.exec(source);
                type = Token.WORD;
                if(result == null){//eof
                    str = source.substring(index,source.length);
                    index01 = source.length;
                }
                else{//
                    str = source.substring(index,result.index);
                    index01 = result.index;//次の開始位置
                }
                addToken(str,type);
                return index01;
                
        }
        
        
        //数値
        if(source.charCodeAt(index) > 47 && source.charCodeAt(index) < 58){
            reg = /[^a-zA-Z0-9_.]/mg;
            reg.lastIndex = index+1;
            result = reg.exec(source);
            type = Token.NUMBER;
            if(result == null){//eof
                str = source.substring(index,source.length);
                index01 = source.length;
            }
            else{// 
                str = source.substring(index,result.index);
                index01 = result.index;//次の開始位置
            }
            //num();//種類判別
            addToken(str,type);
            return index01;

        }
        
        //半角記号
        if(source.charCodeAt(index)<127 && source.charCodeAt(index)>32){
            type = Token.MARK;
            str = source.charAt(index);
            index01 = index+1;
            
            addToken(str,type);
            return index01;
        }
    
        //その他文字列　マルチバイト文字等はここに入る
        type = Token.OTHER;
        reg = /[\x01-\x7f]/mg;
        reg.lastIndex = index+1;
        result = reg.exec(source);
        if(result == null){//eof
            str = source.substring(index,source.length);
            index01 = source.length;
        }
        else{// 
            str = source.substring(index,result.index);
            index01 = result.index;//次の開始位置
        }
        
        addToken(str,type);
        return index01;
    }
     
}//class