forked from: forked from: forked from: forked from: getToken

/**
 * Copyright tepe ( http://wonderfl.net/user/tepe )
 * MIT License ( http://www.opensource.org/licenses/mit-license.php )
 * Downloaded from: http://wonderfl.net/c/1kzy
 */

package {
    //import flash.utils.Dictionary;
    import flash.display.*;
    import flash.text.*;
    import flash.events.*;
    import flash.net.*;
    import flash.utils.*;
    import net.hires.debug.Stats;
    


    public class FlashTest extends Sprite {
        private var t1:TextField = new TextField();//原文
        private var t2:TextField = new TextField();
        private var so : SharedObject;
        public function FlashTest() {
            
            // write as3 code here..
            init();
            //FPS計測    
            addChild( new Stats() );
            //Wonderfl.capture(stage);
            onChange(null);
            
        }
        
 
        private function init():void{
            //原文テキスト入力ボックス
            t1.border = true;
            t1.width = 230;
            t1.height = 360;
            t1.y=100;
            t1.type = "input";
            t1.multiline = true;
            so = SharedObject.getLocal("data");
            if(so){ 
                if(so.data.text != null){
                    t1.text = so.data.text;
                }
                else{    
                    t1.text "";
                    so.data.text = t1.text;
                }
            }
            addChild(t1);
            t1.addEventListener(Event.CHANGE,onChange);
            t2.type ="input";
            t2.x = 230;
            t2.width=230;
            t2.height=460;
            t2.wordWrap =true;
            t2.multiline=true;
            t2.border = true;
            addChild(t2);            
            
        }
        
        

        private var tk:Tokenizer;
        private function onChange(e:Event=null):void{
            
            
            tk = new Tokenizer();
            tk.addEventListener(Event.COMPLETE,onComplete);
            //進捗率の表示
            tk.addEventListener(ProgressEvent.PROGRESS,function(e:ProgressEvent):void{
                t2.text = Math.round(100*e.bytesLoaded/e.bytesTotal).toString()+"%";
            });

            tk.createToken(t1.text);
            //var cnt:int=0;
            t2.text = "";
        }
        
        private var scopeList:Array;
        private var stepList:Array;
        //ステップリストとスコープリストの生成
        private function onComplete(e:Event):void{
            t2.text = "onComplete";

            scopeList = new Array();
            stepList = new Array();
            
            var scopeStack:Array = new Array();
            var scope:Scope = new Scope()//ルートスコープ
            scope.setArea(tk.getToken(),tk.getEof());
            scopeStack.push(scope);
            
            var cToken:Token = tk.getToken();
            //ステップ開始位置を保持するスタックを用意。ステップオブジェクトを生成したら次の開始位置に更新する
            var entryPosStack:Array = new Array();
            var entryPos:Token = cToken;
            var parent:Scope = scope;
            entryPosStack.push(entryPos);
            
            while(cToken!=null){//無限ループ注意
                //記号の処理
                if(cToken.type == Token.MARK){
                    var s:String = cToken.text;
                    if(s == ";"){ 
                        
                        var step:Step = new Step();
                        step.setArea(entryPos,cToken);//前回の位置から今の位置までをステップ区間とする
                        step.setParent(parent);//所属スコープを登録
                        parent.addStep(step);
                        stepList.push(step);//ステップリストに追加
                        //前のステップがあれば連結する
                        entryPos = cToken.next;//開始位置を更新する
                        
                    }
                    else if(s=="{"){ //スコープ開始 
                        
                        //ステップオブジェクトの処理
                        entryPosStack.push(entryPos);
                        entryPos = cToken.next;//新しい階層のステップ開始位置を設定する
                        
                        //スコープオブジェクトの処理
                        var scope2:Scope = new Scope();
                        scope2.setEntry(cToken);//スコープ開始位置
                        scope2.setParent(parent);//上位スコープ指定
                        scopeStack.push(scope2);
                        parent = scopeStack[scopeStack.length-1];//所属スコープ更新
                        
                        
                        
                    }
                    else if(s=="}"){//スコープ終了
                        var scope3:Scope = scopeStack.pop();//上位スコープに戻る
                        parent = scopeStack[scopeStack.length-1];//所属スコープ更新
                        scope3.setArea(scope3.entry,cToken);//スコープ区間の指定
                        scopeList.push(scope3);//スコープリストに追加
                        entryPos = entryPosStack.pop();//一つ上の階層に戻る＞＞
                        
                        var step2:Step = new Step();
                        step2.setArea(entryPos,cToken);//上の階層の前回の位置から今の位置までをステップ区間とする
                        step2.setParent(parent);//上位スコープ
                        step2.setScope(scope3);//下位スコープ
                        stepList.push(step2);//ステップリストに追加
                        parent.addStep(step2);//前のステップがあれば連結する
                        
                        entryPos = cToken.next;//開始位置を更新する
                    }
                    
                }
                cToken = cToken.next;
                //cnt++;
            }
            
            scopeList.push(scopeStack.pop());//グローバルスコープ
            
            var i:int;
            var bufStr:String = new String();
            var root:Scope = scopeList[scopeList.length-1];
            var step5:Step = root.getStep();
            for(i=0;i<root.stepLength;i++){ 
                if(step5.scope!=null){
                    bufStr+=step5.text2+"{ ... }";
                }
                else{
                    bufStr+=step5.text2+";";
                }

                bufStr += "\n-------------------------------\n";
                step5 = step5.next;
                if(step5==null)break;
            }

            t2.text = bufStr;
            so.data.text = t1.text;
            
        }//function
    }//class
}//package
import flash.events.ProgressEvent;



//トークンオブジェクト
class Token{
    private static var idCnt:int;
    private var tokenId:int;
    private var _text:String;//文字列
    public var type:int;//タイプ
    //public var lineNum:int;//属するプロセスライン
    
    //トークンの並びを保持する
    public var next:Token;
    public var prev:Token;
    private var _scope:Scope;
    public static const ERROR:int = -1;//エラー
    public static const SPACE:int = 0;//スペース
    public static const COMMENT:int = 1;//コメント 
    public static const VALUE:int = 2;//数値
    public static const WORD:int = 3;//識別子
    public static const TEXT:int = 4;//文字列
    public static const MARK:int = 5;//記号
    public static const OTHER:int = 6;//その他（マルチバイトの文字列)
    public static const SCOPE:int = 7;//スコープの置き換え
    public static const EOF:int = 8;//EOF
    
    public function Token(){
        idCnt++;
        tokenId = idCnt;
    }
    public function get id():int{
        return tokenId;
    }
    public function set text(str:String):void{
        _text = new String();
        _text += str;
        
    }
    public function get text():String{
        return _text;
    }


}

class Scope{ 
    public var name:String;
    private var stepList:Array;
    private static var idCnt:int;
    private var scopeId:int;//識別値
    private var _entry:Token;//開始位置のトークン
    private var _exit:Token;//終了位置のトークン
    private var _parent:Scope;//上位スコープ
    private var _length:int;

    public function Scope(){
        idCnt++;
        scopeId = idCnt;
        stepList = new Array()
    }
    public function get id():int{
        return scopeId;
    }
    public function get length():int{
        return _length;
    }
    public function addStep(step:Step):int{ 
        
        stepList.push(step);
        if(1<stepList.length){ 
            stepList[stepList.length-1].prev = stepList[stepList.length-2];
            stepList[stepList.length-2].next = stepList[stepList.length-1];
        }
        return 0;
    }
    
    public function getStep(n:int=0):Step{
        if(stepList.length<=n && n<0)return null;
        return stepList[n];
    }
    
    public function get stepLength():int{
        return stepList.length;
    }



    
    public function get entry():Token{
        return _entry;
    }
    
    public function get exit():Token{
        return _exit;
    }
    public function get text():String{//スコープ内のテキスト取得
        var str:String = new String();
        var t:Token = _entry;
        while(t.next != null){
            str += t.text;
            if(t.id == _exit.id)break;
            t = t.next;
        }
        return str;
    }
    
    public function setEntry(entry:Token):int{
        if(entry.text != "{")return -1;
        _entry = entry;
        return 0;
    }
    public function setParent(parent:Scope):void{
        _parent = parent;
    }
    public function get parent():Scope{
        return _parent;
    }

    
    public function setArea(entry:Token,exit:Token):int{
        _entry = entry;
        _exit = exit;
        _length = 0;
        var t:Token = entry;
        while(true){
            _length++;
            if(t.next == null)break;
            else if(t.id == exit.id)break;
            
            t = t.next;
        }
        return _length;
    }
}
//１ステップに含まれるトークンへの参照
//ステップが所属するスコープ
//ステップのString表現

class Step{ 
    public var name:String;
    private static var idCnt:int;
    private var _entry:Token;//開始位置のトークン
    private var _exit:Token;//終了位置のトークン
    private var _parent:Scope;//上位スコープ
    private var _scope:Scope;//下位スコープ
    private var _length:int;
    private var _prev:Step;//前のステップ
    private var _next:Step;//次のステップ
    private var stepId:int;//ステップの識別値
    
    public function Step(){
        idCnt++;
        stepId = idCnt;
    }
    
    public function get length():int{
        return _length;
    }
    public function get id():int{
        return stepId;
    }
    public function get entry():Token{
        return _entry;
    }
    
    public function get exit():Token{
        return _exit;
    }
    public function get text():String{//スコープ内のテキスト取得
        var str:String = new String();
        var t:Token = _entry;
        while(t.next != null){
            str += t.text;
            if(t.id == _exit.id)break;
            t = t.next;
        }
        return str;
    }
    public function get text2():String{ 
        
        var str:String = new String();
        var t:Token = _entry;
        while(t.type == Token.SPACE || t.type == Token.COMMENT){
            t = t.next;
        }
        var m:String;
        if(scope == null)m=";";
        else m="{";
        while(t.next != null){ 
            if(t.text == m)break;
            str += t.text;
            t = t.next;
        }
        return str;

    }

    
    public function setEntry(entry:Token):int{
        if(entry.text != "{")return -1;
        _entry = entry;
        return 0;
    }
    public function setParent(parent:Scope):void{
        _parent = parent;
    }
    public function get parent():Scope{
        return _parent;
    }

    public function setScope(scope:Scope):void{
        _scope = scope;
    }
    public function get scope():Scope{
        return _scope;
    }


    
    public function set prev(step:Step):void{ 
        _prev = step;
    }
    public function set next(step:Step):void{
        _next = step;
    }
    public function get prev():Step{
        return _prev;
    }
    public function get next():Step{
        return _next;
    }




    
    public function setArea(entry:Token,exit:Token):int{
        _entry = entry;
        _exit = exit;
        _length = 0;
        var t:Token = entry;
        while(true){
            _length++;
            if(t.next == null)break;
            else if(t.id == exit.id)break;
            
            t = t.next;
        }
        return _length;
    }
}



//Thread==========================================================================
import flash.utils.*;
import flash.events.*;
    
class Thread extends Timer{ 
    static public var SPAN:Number = 1000/30; 
    static public var RATE:Number = 0.5;
    public var end:Boolean = false, currentLoop:Loop = null;
    
    static private var NUM:int = 0;
    private var _limit:Number = 5, _time:int = 0, _added:Boolean, currentLoops:Vector.<Loop>, loops:Vector.<Loop> = new Vector.<Loop>();
    
    function Thread(){
        super(SPAN); start(); NUM++;
        addEventListener( "timer", onFrame  );
    }    
    public function loop( func:Function, onComplete:Function = null):void{
        var loop:Loop = new Loop( func, currentLoop );     
        if( currentLoop == null ){ loops.push( loop );
        }else{ currentLoop.loops.push( loop ); _added = true }
        if( onComplete != null ){
            loop = new Loop( onComplete, currentLoop );   
            if( currentLoop == null ){ loops.push( loop );
            }else{ currentLoop.loops.push( loop ); }
        }
    }    
    public function remove():void{
        NUM--; stop(); end = true;
        removeEventListener( "timer", onFrame  );
    }    
    private function onFrame(e:Event):void{
        _time = getTimer(); _limit = (SPAN * RATE) / NUM;
        all: while(true){
            if( currentLoop == null ){
                currentLoops = loops;
                if( loops.length == 0 ){ remove(); break; }
                while( currentLoops[0].loops.length != 0 ){ currentLoops = currentLoops[0].loops;  }
                currentLoop = currentLoops[0];
            }
            do{
                if( _limit < (getTimer() - _time)){ break all; }
                if( _added ){ _added = false; currentLoop = null; continue all;  }
            }while( currentLoop.func() )
            currentLoop = null;
            currentLoops.reverse(); currentLoops.pop(); currentLoops.reverse();
        }
    }
}
class Loop extends Object{
    public var func:Function, name:String, parent:Loop, loops:Vector.<Loop> = new Vector.<Loop>();
    function Loop( func:Function, parent:Loop = null, name:String = "" ){  
        this.func = func; this.parent = parent;
    }
}
//=======================================================================================



//////////////////////////////////////////////////////////////////////////
//  トークン列取得
//////////////////////////////////////////////////////////////////////////



//テキストからトークンを抽出する
class Tokenizer extends Thread{
    private var tokenList:Array;
    private var scopeList:Array;
    private var _length:int;
    private var source:String;
    private var cansel:Boolean;
    public function Tokenizer(){
        
    }
    //スコープリストを生成する
    public function createScopeList():int{
        return 0;
    }

    public function getEof():Token{
        return getToken(length-1);
    }


    //指定インデクスからmax個までトークン抽出
    //抽出したトークンはlistに追加
    //抽出したところまでのインデクスを返す
    public function createToken(str:String):void{
        
        source = str;
        tokenList = new Array();
        index01 = 0;
         
        var i:int = 0;
        var cnt:int = 0;
        loop( 
            function _while():Boolean{ 
                i= func01();
                
                if(cnt==0){
                    var e:ProgressEvent = new ProgressEvent(ProgressEvent.PROGRESS,false,true,i,str.length);
                    dispatchEvent(e);
                }
                cnt++;
                cnt%=500;

                return i<str.length;
            }, 
            function onComplete():void { 
                addToken("",Token.EOF);//EOF追加
                _length = tokenList.length;
                var event:Event = new Event(Event.COMPLETE);
                dispatchEvent(event);
                
            } 
        );
    }
    public function get length():int{
        return _length;
    }
    //先頭からの順を指定してトークンを取得
    public function getToken(index:int=0):Token{
        if(index<0 && tokenList.length<=index)return null;
        return tokenList[index];
    }
    
    //トークン列に追加
    private function addToken(str:String,type:int):int{
        var obj:Token = new Token();
        obj.text = str;
        obj.type = type;
        obj.next = null;
        tokenList.push(obj);
        if(1<tokenList.length){ 
            tokenList[tokenList.length-1].prev = tokenList[tokenList.length-2];
            tokenList[tokenList.length-2].next = tokenList[tokenList.length-1];
        }
        else tokenList[tokenList.length-1].prev = null;
        return tokenList.length;
    }



//-------------------------------------------------------------
    //開始インデクスを受け取りトークンのタイプを返す
    private var index01:int=0;
    private function func01():int{//
        if(index01 >= source.length)return source.length;
        var index:int = index01;
        var type:int;//トークンタイプ
        var str:String;//トークン文字列
        var reg:RegExp = new RegExp();                
        var result:Object;
        //トークンタイプ：空白か？
        if(    source.charAt(index)== " " ||
               source.charAt(index)=="\t" ||
               source.charAt(index)=="\n" ||
               source.charAt(index)=="\r" ){
                   type = Token.SPACE;
                   reg = /[^\s]/mg;
                   reg.lastIndex = index;
                   result = reg.exec(source);
                   //トークン作成
                   if(result == null){//
                       str = source.substring(index,source.length);
                       index01 = source.length;//次の開始位置
                   }
                   else{//
                       str = source.substring(index,result.index);
                       index01 = result.index;                       
                   }
                   addToken(str,type);
                   return index01;
                                       
               }

        
        //トークンタイプ:文字列定数１
        if(source.charAt(index)== "'"){
            type = Token.TEXT;
            var indexA:int = index;
            reg = /['\n\r]/mg;//改行か次の[']が現れる位置
            do{//エスケープ処理
                reg.lastIndex = indexA+1;
                result = reg.exec(source);
                if(result==null)break;
                else indexA = result.index;
            }while(source.charAt(indexA-1)=="\\");//["]が現れてもその前がエスケープなら再検索
            
            //トークン作成
            if(result == null){//エラー
                str = "'";
                type = Token.ERROR;
                index01 = index+1;//次の開始位置
            }
            else{//
                str = source.substring(index,result.index+1);
                index01 = result.index+1;//次の開始位置
            }
            addToken(str,type);
            return index01;
                
        }
 
        //トークンタイプ:文字列定数2
        if(source.charAt(index)== '"'){
            type = Token.TEXT;
            var indexB:int = index;
            reg = /["\n\r]/mg;//改行か次の["]が現れる位置
            do{//エスケープ処理
                reg.lastIndex = indexB+1;
                result = reg.exec(source);
                if(result==null)break;
                else indexB = result.index;
            }while(source.charAt(indexB-1)=="\\");//["]が現れてもその前がエスケープなら再検索
            
            //トークン作成
            if(result == null){//エラー
                str = '"';
                type = Token.ERROR;
                index01 = index+1;//次の開始位置
                
            }
            else{//
                str = source.substring(index,result.index+1);
                index01 = result.index+1;//次の開始位置
                
            }
            addToken(str,type);
            return index01;
        }
        
        if(source.charAt(index)== "/"){
               //一行コメント
               if(source.charAt(index+1)== "/"){//
                   reg = /[\n\r]/mg;
                   reg.lastIndex = index+2;
                   result = reg.exec(source);
                   //トークン作成
                   type = Token.COMMENT;
                   if(result == null){
                       str = source.substring(index,source.length);
                       index01 = source.length;//次の開始位置                       
                   }
                   else{// 
                       str = source.substring(index,result.index);
                       index01 = result.index;//次の開始位置
                   }
                   addToken(str,type);
                   return index01;
               }
               //複数行コメント
               if(source.charAt(index+1)== "*"){
                   reg = /\*\//mg;
                   reg.lastIndex = index+2;
                   result = reg.exec(source);
                   
                   //トークン作成
                   type = Token.COMMENT;
                   if(result == null){//エラー
                       type = Token.ERROR;
                       str = '/*';
                       index01 = index+2;//次の開始位置
                   }
                   else{//
                       str = source.substring(index,result.index+2);
                       index01 = result.index+2;//次の開始位置                       
                   }
                   addToken(str,type);
                   return index01;
               }
        }

        //英数字
        if((source.charCodeAt(index) > 64 && source.charCodeAt(index) < 91) ||// A-Z
           (source.charCodeAt(index) > 96 && source.charCodeAt(index) < 123)||// a-z
            source.charCodeAt(index) == 95 ){// _
                reg = /[^a-zA-Z0-9_]/mg;//英数字以外が現れる位置
                reg.lastIndex = index+1;
                result = reg.exec(source);
                type = Token.WORD;
                if(result == null){//eof
                    str = source.substring(index,source.length);
                    index01 = source.length;
                }
                else{//
                    str = source.substring(index,result.index);
                    index01 = result.index;//次の開始位置
                }
                addToken(str,type);
                return index01;
                
        }
        
        
        //数値
        if(source.charCodeAt(index) > 47 && source.charCodeAt(index) < 58){
            reg = /[^a-zA-Z0-9_.]/mg;
            reg.lastIndex = index+1;
            result = reg.exec(source);
            type = Token.VALUE;
            if(result == null){//eof
                str = source.substring(index,source.length);
                index01 = source.length;
            }
            else{// 
                str = source.substring(index,result.index);
                index01 = result.index;//次の開始位置
            }
            //num();//種類判別
            addToken(str,type);
            return index01;

        }
        
        //半角記号
        if(source.charCodeAt(index)<127 && source.charCodeAt(index)>32){
            type = Token.MARK;
            str = source.charAt(index);
            index01 = index+1;
            
            addToken(str,type);
            return index01;
        }
    
        //その他文字列　マルチバイト文字等はここに入る
        type = Token.OTHER;
        reg = /[\x01-\x7f]/mg;
        reg.lastIndex = index+1;
        result = reg.exec(source);
        if(result == null){//eof
            str = source.substring(index,source.length);
            index01 = source.length;
        }
        else{// 
            str = source.substring(index,result.index);
            index01 = result.index;//次の開始位置
        }
        
        addToken(str,type);
        return index01;
    }
     
}//class