#include <stdio.h>
#include <time.h>
#include <math.h>
#include "NeuralNetwork.h"
#include "Muele.h"
#include "TestWin.h"
#include "Symmetry.h"
#include "NeuralNetworkWindow.h"
#include "PreferencesWindow.h"

//#define XOR_TEST

CNeuralNetwork::CNeuralNetwork(int boardSizeX, int boardSizeY, int boardSizeZ, int winFieldsCount, int countHiddenNeurons, bool largeInputVector, CMuele *pMuele) :
	numberHiddenNeurons(countHiddenNeurons), boardFieldCount(boardSizeX * boardSizeY * boardSizeZ), 
	boardSizeX(boardSizeX), boardSizeY(boardSizeY), boardSizeZ(boardSizeZ), largeInputVector(largeInputVector),
	winFieldCount(winFieldsCount), pMuele(pMuele), gamesCount(0), actualCycle(0), boardFieldCountPlain(boardSizeX * boardSizeY)
{
#ifdef XOR_TEST
	boardFieldCount = 2;
	numberHiddenNeurons=3;
	numberInputNeurons=3;
	allocateMemory();
	resetWeights(PLAYER_1);
    xorTest();
	exit(0);
#endif

	numberInputNeurons=boardFieldCount;
	if ( largeInputVector ) numberInputNeurons*=3;
	numberInputNeurons++;    // extra bias(treshold) neuron
	numberHiddenNeurons++;   // extra bias neuron

	trainRateEnd=trainRateStart=maxErrorEnd=maxErrorStart=stepSizeEnd=stepSizeStart=0.0;
	trainingTime=0;
	wasTrained=false;

	allocateMemory();

	resetWeights(PLAYER_1);
	resetWeights(PLAYER_2);

	srand( (unsigned)time( NULL ) );
}

/********************************************************************/

void CNeuralNetwork::resetWeights(char player)
{
	double *pWeightsHidden;
	double *pWeightsOut;

	if ( player == PLAYER_1 )
	{
		pWeightsHidden = pWeightsHidden1;
		pWeightsOut = pWeightsOut1;
	}
	else
	{
		pWeightsHidden = pWeightsHidden2;
		pWeightsOut = pWeightsOut2;
	}

	//float randomDecrease = 1.0;
	float maxAbsWightSize = 1.0;

	//  reset weights between input and hidden layer
	for ( int i = 0; i < numberHiddenNeurons * numberInputNeurons; i++ )
			//pWeightsHidden[i] = 0.5 / randomDecrease - rand() / float(RAND_MAX * randomDecrease);
			pWeightsHidden[i] = 2.0 * ( rand() / float(RAND_MAX) - 0.5 ) * maxAbsWightSize ;

	// reset weights between hidden and output layer
	for ( int i = 0; i < numberHiddenNeurons; i++ )
	{
		//pWeightsOut[i] = 0.5 / randomDecrease - rand() / float(RAND_MAX * randomDecrease);
		pWeightsOut[i] = 2.0 * ( rand() / float(RAND_MAX) - 0.5 ) * maxAbsWightSize ;

		// set bias weights to 0
		pWeightsHidden[i * numberInputNeurons + numberInputNeurons-1] = 0.0;
	}
	
	// set bias weight to 0
	pWeightsOut[numberHiddenNeurons - 1] = 0.0;

	// initialise output for treshold neurons which doesn't change
	pOutputHidden[numberHiddenNeurons - 1] = -1.0; //bias, treshold
	pInputVector[numberInputNeurons - 1] = -1.0;  // bias neuron of input layer

}


/********************************************************************/

CNeuralNetwork::~CNeuralNetwork(void)
{
	deleteUsedMemory();
}

/********************************************************************/

void CNeuralNetwork::allocateMemory()
{
	pWeightsHidden1 = new double[ numberInputNeurons * numberHiddenNeurons];
	pWeightsHidden2 = new double[ numberInputNeurons * numberHiddenNeurons];
	pWeightsOut1 = new double[ numberHiddenNeurons ];
	pWeightsOut2 = new double[ numberHiddenNeurons ];

	pInputVector = new double[ numberInputNeurons ];
	pOutputHidden = new double [ numberHiddenNeurons ];
	pGradientsHidden = new double[ numberHiddenNeurons ];

	pSymmetry = new CSymmetry( boardSizeX, boardSizeY, boardSizeZ );

	pBoard  = new char[boardFieldCount];

	pTestWinPlayer1 = new CTestWin( pBoard , NULL, boardSizeX, boardSizeY, boardSizeZ, PLAYER_1, winFieldCount, 1);
	pTestWinPlayer2 = new CTestWin( pBoard , NULL, boardSizeX, boardSizeY, boardSizeZ, PLAYER_2, winFieldCount, 1);
}

/********************************************************************/

void CNeuralNetwork::deleteUsedMemory()
{
	delete [] pWeightsHidden1;
	delete [] pWeightsOut1;
	delete [] pWeightsHidden2;
	delete [] pWeightsOut2;

	delete [] pOutputHidden;
	delete [] pInputVector;
	delete [] pGradientsHidden;
	
	delete pSymmetry;

	delete [] pBoard;
	
	delete pTestWinPlayer1;
	delete pTestWinPlayer2;
}

/********************************************************************/

bool CNeuralNetwork::saveNet(const wchar_t *pFileName)
{
	core::stringc fileName;

	fileName = boardSizeX;
	fileName += boardSizeY;
	fileName += boardSizeZ;
	fileName +="_";
	fileName += winFieldCount;
	if ( pFileName != NULL )
	{
		fileName+="_";
		fileName+=numberHiddenNeurons;
		if ( largeInputVector ) fileName+="_li";
		fileName+="_";
		fileName+=pFileName;
	}
	fileName += ".net";

	io::IFileSystem *fileSystem = pMuele->pDevice->getFileSystem();
	io::IWriteFile *fileOut = fileSystem->createAndWriteFile(fileName.c_str(), false);

	if ( fileOut == NULL ) return false;

	unsigned long fileMark = FILE_MARK;

	fileOut->write( &fileMark, sizeof(unsigned long) );
	fileOut->write( &numberInputNeurons, sizeof(int) );
	fileOut->write( &numberHiddenNeurons, sizeof(int) );
	fileOut->write( &largeInputVector, sizeof(bool) );
	fileOut->write( &gamesCount, sizeof(int) );
	fileOut->write( &actualCycle, sizeof(int) );
	fileOut->write( &boardSizeX, sizeof(int) );
	fileOut->write( &boardSizeY, sizeof(int));
	fileOut->write( &boardSizeZ, sizeof(int));
	fileOut->write( &winFieldCount, sizeof(int));

	fileOut->write( &drawCount, sizeof(unsigned));
	fileOut->write( &win1Count, sizeof(unsigned));
	fileOut->write( &win2Count, sizeof(unsigned));

	fileOut->write( &maxErrorStart, sizeof(double));
	fileOut->write( &maxErrorEnd, sizeof(double));
	fileOut->write( &trainRateStart, sizeof(double));
	fileOut->write( &trainRateEnd, sizeof(double));
	fileOut->write( &stepSizeStart, sizeof(double));
	fileOut->write( &stepSizeEnd, sizeof(double));
	fileOut->write( &trainingTime, sizeof(unsigned));
	fileOut->write( &maxDepth, sizeof(int));
	fileOut->write( &netRaiting, sizeof(int));

	fileOut->write( pWeightsHidden1, sizeof(double) * numberInputNeurons * numberHiddenNeurons );
	fileOut->write( pWeightsHidden2, sizeof(double) * numberInputNeurons * numberHiddenNeurons );
	fileOut->write( pWeightsOut1, sizeof(double) * numberHiddenNeurons );
	fileOut->write( pWeightsOut2, sizeof(double) * numberHiddenNeurons );

	fileOut->drop();
	
	core::stringw out = "          name: ";
	out+=fileName.c_str();
	pMuele->pNeuralNetworkWindow->pTextName->setText(out.c_str());
	pMuele->pNeuralNetworkWindow->resetWindow(true);

	wasTrained=false;

	return true;
}

/********************************************************************/

bool CNeuralNetwork::loadNet(const wchar_t *pFileName)
{
	core::stringc fileName;

	if ( pFileName == NULL )
	{
		fileName = boardSizeX;
		fileName += boardSizeY;
		fileName += boardSizeZ;
		fileName +="_";
		fileName += winFieldCount;
		fileName += ".net";
	}
	else fileName = pFileName;

	io::IFileSystem *fileSystem = pMuele->pDevice->getFileSystem();
	io::IReadFile *fileIn = fileSystem->createAndOpenFile(fileName.c_str());

	if ( fileIn == NULL ) return false;

	unsigned long fileMark;

	// neural network files have a certain long value at first, check if this value is correct
	fileIn->read( &fileMark, sizeof(unsigned long) );
	if ( fileMark != FILE_MARK )
	{
		// the file is no neural network file
		fileIn->drop();
		return false;
	}

	fileIn->read( &numberInputNeurons, sizeof(int) );
	fileIn->read( &numberHiddenNeurons, sizeof(int) );
	fileIn->read( &largeInputVector, sizeof(bool) );
	fileIn->read( &gamesCount, sizeof(int) );
	fileIn->read( &actualCycle, sizeof(int) );
	fileIn->read( &boardSizeX, sizeof(int) );
	fileIn->read( &boardSizeY, sizeof(int));
	fileIn->read( &boardSizeZ, sizeof(int));
	fileIn->read( &winFieldCount, sizeof(int));

	fileIn->read( &drawCount, sizeof(unsigned));
	fileIn->read( &win1Count, sizeof(unsigned));
	fileIn->read( &win2Count, sizeof(unsigned));

	fileIn->read( &maxErrorStart, sizeof(double));
	fileIn->read( &maxErrorEnd, sizeof(double));
	fileIn->read( &trainRateStart, sizeof(double));
	fileIn->read( &trainRateEnd, sizeof(double));
	fileIn->read( &stepSizeStart, sizeof(double));
	fileIn->read( &stepSizeEnd, sizeof(double));
	fileIn->read( &trainingTime, sizeof(unsigned));
	fileIn->read( &maxDepth, sizeof(int));
	fileIn->read( &netRaiting, sizeof(int));

	boardFieldCount = boardSizeX * boardSizeY * boardSizeZ;
	boardFieldCountPlain = boardSizeX * boardSizeY;

	pMuele->winFieldCount = winFieldCount;
	if ( pMuele->boardSizeX != boardSizeX || pMuele->boardSizeY != boardSizeY || pMuele->boardSizeZ != boardSizeZ )
	{
		pMuele->boardSizeX = boardSizeX;
		pMuele->boardSizeY = boardSizeY;
		pMuele->boardSizeZ = boardSizeZ;
		pMuele->resetBoard(true);
	}
	pMuele->pPreferencesWindow->resetWindow();

	deleteUsedMemory();
	allocateMemory();

	fileIn->read( pWeightsHidden1, sizeof(double) * numberInputNeurons * numberHiddenNeurons );
	fileIn->read( pWeightsHidden2, sizeof(double) * numberInputNeurons * numberHiddenNeurons );
	fileIn->read( pWeightsOut1, sizeof(double) * numberHiddenNeurons );
	fileIn->read( pWeightsOut2, sizeof(double) * numberHiddenNeurons );
	
	fileIn->drop();

	core::stringw out = "          name: ";
	out+=fileName.c_str();
	pMuele->pNeuralNetworkWindow->pTextName->setText(out.c_str());
	pMuele->pNeuralNetworkWindow->resetWindow(true);

	wasTrained=false;

	return true;
}

/********************************************************************/

void CNeuralNetwork::createInputVector(const char *pBoard)
{

#ifdef XOR_TEST

	if ( pBoard[0] == 0) pInputVector[0]=0; else pInputVector[0]=1;
	if ( pBoard[1] == 0) pInputVector[1]=0; else pInputVector[1]=1;

	return;
#endif

	//memset(pInputVector, 0, sizeof(double) * numberInputNeurons);
	for ( int i = 0; i < numberInputNeurons; i++ )
		pInputVector[i] = -1.0;

	if ( largeInputVector )
	{
		for ( int boardField = 0; boardField < boardFieldCount; boardField++ )
		{
			switch ( pBoard[boardField] )
			{
			case EMTY_FIELD: pInputVector[boardField * (PLAYER_2+1) + EMTY_FIELD] = 1; break;
			case PLAYER_1:   pInputVector[boardField * (PLAYER_2+1) + PLAYER_1]   = 1; break;
			case PLAYER_2:   pInputVector[boardField * (PLAYER_2+1) + PLAYER_2]   = 1; break;
			}
		}
	}
	else
	{
		for ( int boardField = 0; boardField < boardFieldCount; boardField++ )
		{
			switch ( pBoard[boardField] )
			{
			case EMTY_FIELD: pInputVector[boardField] = 0; break;
			case PLAYER_1:   pInputVector[boardField] = -1; break;
			case PLAYER_2:   pInputVector[boardField] = 1; break;
			}
		}
	}
}

/********************************************************************/

double CNeuralNetwork::propagate(char *pBoard, char player, bool createTheInputVector )
{
	double nettoInput;
	double *pWeightsHidden;
	double *pWeightsOut;

	if ( player == PLAYER_1 )
	{
		pWeightsHidden = pWeightsHidden1;
		pWeightsOut = pWeightsOut1;
	}
	else
	{
		pWeightsHidden = pWeightsHidden2;
		pWeightsOut = pWeightsOut2;
	}

	if ( createTheInputVector ) createInputVector( pBoard );

	// numberHiddenNeurons - 1 because Bias output is always -1, was set in constructor
	for ( int hiddenNeuron = 0; hiddenNeuron < numberHiddenNeurons-1; hiddenNeuron++ )
	{
		nettoInput = 0;
		// go throught all input neurons
		for ( int inputNeuron = 0; inputNeuron < numberInputNeurons; inputNeuron++ )
			nettoInput+=*(pInputVector + inputNeuron) * *(pWeightsHidden + hiddenNeuron * numberInputNeurons + inputNeuron);

		// netto input calculated 
		// calculate output of hidden neuron
		*(pOutputHidden + hiddenNeuron) = 1.0 / (1.0 + exp(-nettoInput)); // logistic activation
	}

	// calculate output value of the one output neuron
	// inclusive bias
	nettoInput = 0;
	for ( int hiddenNeuron = 0; hiddenNeuron < numberHiddenNeurons; hiddenNeuron++ )
		nettoInput+=*(pOutputHidden + hiddenNeuron) * *(pWeightsOut + hiddenNeuron);

	return 1.0 / (1.0 + exp(-nettoInput));
}



/********************************************************************/

void CNeuralNetwork::trainValue(char *pBoard, char player, double trainValue, double learnRate, double maxError)
{
	double *pWeightsHidden;
	double *pWeightsOut;

	if ( player == PLAYER_1 )
	{
		pWeightsHidden = pWeightsHidden1;
		pWeightsOut = pWeightsOut1;
	}
	else
	{
		pWeightsHidden = pWeightsHidden2;
		pWeightsOut = pWeightsOut2;
	}
	
	double actualOutput;
	double gradientOutput;

	createInputVector( pBoard );

	// propagation also calculates output hidden layer
	actualOutput = propagate(pBoard, player, false);         

	int maxTrainCycles = 0;

	while ( fabs( trainValue - actualOutput ) > maxError )
	{
		if ( maxTrainCycles++ > 2000) break;

		// gradient output layer
		gradientOutput = actualOutput * ( 1 - actualOutput) * (trainValue - actualOutput);
		
		// gradients hidden layer
		for ( int hiddenNeuron = 0; hiddenNeuron < numberHiddenNeurons-1; hiddenNeuron++ )
			pGradientsHidden[hiddenNeuron] = pOutputHidden[hiddenNeuron] * ( 1 - pOutputHidden[hiddenNeuron])
				* gradientOutput * pWeightsOut[hiddenNeuron];
		
		// weight changes output
		for ( int hiddenNeuron = 0; hiddenNeuron < numberHiddenNeurons; hiddenNeuron++ )
			*(pWeightsOut + hiddenNeuron)+= learnRate * *(pOutputHidden + hiddenNeuron ) * gradientOutput;

		// weight changes hidden
		for ( int hiddenNeuron = 0; hiddenNeuron < numberHiddenNeurons - 1; hiddenNeuron++ )
			for ( int inputNeuron = 0; inputNeuron < numberInputNeurons; inputNeuron++)
				*(pWeightsHidden + hiddenNeuron * numberInputNeurons + inputNeuron)+= learnRate * *(pInputVector + inputNeuron) * *(pGradientsHidden + hiddenNeuron);

		actualOutput = propagate(pBoard, player, false);         // calculates output hidden
	}
}

/********************************************************************/

int CNeuralNetwork::getBestMove(char *pBoard, char player)
{
	double bestStateValue = -10000.0;
	int bestFieldNumber;
	double propagateVal;

	for ( int i = 0; i < boardFieldCountPlain; i++ )
		for ( int j = 0; j < boardFieldCount; j+=boardFieldCountPlain )
			if ( pBoard[j + i] == EMTY_FIELD )
			{
				pBoard[j + i] = player;
				propagateVal = propagate(pSymmetry->getSymmetry(pBoard), player);
				
				if ( bestStateValue < propagateVal )
				{
					bestStateValue = propagateVal;
					bestFieldNumber = j + i;
				}
				pBoard[j + i] = EMTY_FIELD;
				break;
			}

	return bestFieldNumber;
}

/********************************************************************/

void CNeuralNetwork::aICompetition( char weakPlayer, int testCount, int &wonPlayer1, int &draws, int &wonPlayer2)
{
	char actualPlayer;
	int move;
	wonPlayer1 = wonPlayer2 = draws = 0;
	int randomField;
 	
	srand( (unsigned)time( NULL ) );

	for ( long loopCount = 0; loopCount < testCount; loopCount ++ )
	{
		memset(pBoard, 0, boardFieldCount );
		actualPlayer = PLAYER_1;

		// play one complete game
		for ( move = 0; move < boardFieldCount; move++ )
		{
			if ( weakPlayer == actualPlayer && ( rand() % (( boardFieldCount ) - ( move )  ) == 0) )
			{
				// make a random move
				for ( int j = 0; j < boardFieldCountPlain; j++ )
				{
					randomField = (rand()%(boardFieldCountPlain));
					for ( int k = randomField; k < boardFieldCount; k+=boardFieldCountPlain )
						if ( pBoard[k] == EMTY_FIELD )
						{
							pBoard[k] = actualPlayer;
							goto weiter;
						}
				}
				for ( int j = 0; j < boardFieldCountPlain; j++ )
					for ( int k = j; k < boardFieldCount; k+=boardFieldCountPlain )
						if ( pBoard[k] == EMTY_FIELD) 
						{
							pBoard[k] = actualPlayer;
							goto weiter;
						}
				weiter:;
			}
			else // make suggested move
			{
				pBoard[getBestMove(pBoard, actualPlayer)] = actualPlayer;
			}

			// test for a winner
			if ( actualPlayer == PLAYER_1 )
			{
				if ( pTestWinPlayer1->testWin() )
				{
					wonPlayer1++;
					break;
				}
			}
			else
			{
				if ( pTestWinPlayer2->testWin() )
				{
					wonPlayer2++;
					break;
				}
			}

			if ( actualPlayer == PLAYER_1 ) actualPlayer = PLAYER_2; else actualPlayer = PLAYER_1; 
		} // play one complete game loop
		if ( move == boardFieldCount ) draws++;
	}
   //	delete[] pBoard;
}

/********************************************************************/

void CNeuralNetwork::setLearnParameters()
{
	core::stringc s;

	s = pMuele->pNeuralNetworkWindow->pEditTrainCylcles->getText();
	gamesCount = actualCycle = atoi(s.c_str());
	if ( gamesCount < 100 ) gamesCount = actualCycle = 100;

	s = pMuele->pNeuralNetworkWindow->pEditMaxErrorStart->getText();
	maxErrorStart = atof(s.c_str());
	if ( maxErrorStart == 0 ) maxErrorStart = 0.001;

	s = pMuele->pNeuralNetworkWindow->pEditMaxErrorEnd->getText();
	maxErrorEnd = atof(s.c_str());
	if ( maxErrorEnd > maxErrorStart ) maxErrorEnd = maxErrorStart;

	s = pMuele->pNeuralNetworkWindow->pEditTrainRateStart->getText();
	trainRateStart = atof(s.c_str());
	if ( trainRateStart == 0 ) trainRateStart = 0.2;

	s = pMuele->pNeuralNetworkWindow->pEditTrainRateEnd->getText();
	trainRateEnd = atof(s.c_str());
	if ( trainRateEnd > trainRateStart ) trainRateEnd = trainRateStart;

	s = pMuele->pNeuralNetworkWindow->pEditStepSizeStart->getText();
	stepSizeStart = atof(s.c_str());
	if ( stepSizeStart == 0 ) stepSizeStart = 0.1;

	s = pMuele->pNeuralNetworkWindow->pEditStepSizeEnd->getText();
	stepSizeEnd = atof(s.c_str());
	if ( stepSizeEnd > stepSizeStart ) stepSizeEnd = stepSizeStart;

	strengthCount = 0;
	drawCount = win1Count = win2Count = 0;
	trainingTime = netRaiting = 0;
	maxDepth = 2 * winFieldCount;
	maxDepthCount = 0;
	oldCyclesPlayer1 = oldCyclesPlayer2 = 1;
}

/********************************************************************/
/* normal stepwise temporal differences learning					*/

void CNeuralNetwork::learn()
{

	register char *pMoves  = new char[boardFieldCount];

	char actualPlayer, lastPlayer;    	
	int move;

	double positiveReward = 0.8;
	double negativeReward = 0.2;
	double maxError, learnRate;
	double alpha;
	int maxDepthLocal = winFieldCount * 2 - 1;

	wasTrained = true;
	
	u32 actualTime = pMuele->pTimer->getRealTime();

	int lastMove;
	
	for ( ; actualCycle > 0; actualCycle--)
	{
		maxError = (double)actualCycle / gamesCount * ( maxErrorStart - maxErrorEnd ) + maxErrorEnd;
		learnRate = (double)actualCycle / gamesCount * ( trainRateStart - trainRateEnd ) + trainRateEnd;
		alpha = (double)actualCycle / gamesCount * ( stepSizeStart - stepSizeEnd ) + stepSizeEnd;

		memset(pBoard, 0, boardFieldCount );
		actualPlayer = PLAYER_1;

		// play one complete game
		for ( move = 0;  ; move++ )
		{
			if ( move > 2 )
			{
				double valueNow = propagate(pSymmetry->getSymmetry(pBoard), lastPlayer);
				char savePlayera = pBoard[lastMove];
				pBoard[lastMove]=EMTY_FIELD;
				char savePlayerb = pBoard[pMoves[move-2]];
				pBoard[pMoves[move-2]]=EMTY_FIELD;
				double valueBefore = propagate(pSymmetry->getSymmetry(pBoard), lastPlayer);
			
				trainValue(pSymmetry->getSymmetry(pBoard), lastPlayer, valueBefore + ( valueNow - valueBefore) * alpha , learnRate, maxError);

				pBoard[lastMove]=savePlayera;
				pBoard[pMoves[move-2]]=savePlayerb;
			}

			// If actual player can win, take the winning move.
			for ( int j = 0; j < boardFieldCountPlain; j++ )
				for ( int k = j; k < boardFieldCount; k+=boardFieldCountPlain )
					if ( pBoard[k] == EMTY_FIELD )
					{
						pBoard[k] = actualPlayer;
						if ( (actualPlayer == PLAYER_1 && pTestWinPlayer1->testWin()) || (actualPlayer == PLAYER_2 && pTestWinPlayer2->testWin()) )
						{
							pMoves[move] = k;
							lastMove = k;
							if ( actualPlayer == PLAYER_1 ) win1Count++; else win2Count++;

							trainValue(pSymmetry->getSymmetry(pBoard), actualPlayer, positiveReward, learnRate, maxError);
							pBoard[lastMove]=EMTY_FIELD;
							trainValue(pSymmetry->getSymmetry(pBoard), lastPlayer, negativeReward, learnRate, maxError);
							pBoard[pMoves[move-1]]=EMTY_FIELD;
							double valueWinBef = propagate(pSymmetry->getSymmetry(pBoard), actualPlayer);
							trainValue(pSymmetry->getSymmetry(pBoard), actualPlayer, valueWinBef + ( positiveReward - valueWinBef) * alpha, learnRate, maxError);
							pBoard[pMoves[move-2]]=EMTY_FIELD;
							double valueLoseBef = propagate(pSymmetry->getSymmetry(pBoard), lastPlayer);
							trainValue(pSymmetry->getSymmetry(pBoard), lastPlayer, valueLoseBef + ( negativeReward - valueLoseBef) * alpha, learnRate, maxError);
							goto wonGame;
						}
						pBoard[k] = EMTY_FIELD;
						break;
					}
			
//			 make a random move or the best move
//			int prop;
//			if ( maxDepth > move ) prop = ( maxDepth - move ) + 1; else prop =maxDepth/2;
//			if ( mt_random() % ( prop ) == 0)
			if ( rand() % ( 1 + ( boardFieldCount ) - ( move )  ) == 0 )
//			if ( mt_random() % ( 1 + ( maxDepth ) - ( move )  ) == 0 )
//			if ( mt_random() % ( boardFieldCount / 2  ) == 0 )

			{  
				// exploring move
				int randomField;
				for ( int j = 0; j < boardFieldCountPlain; j++ )
				{
					//randomField =(rand()%(boardFieldCountPlain));
					randomField =(rand()%(boardFieldCountPlain));
					for ( int k = randomField; k < boardFieldCount; k+=boardFieldCountPlain )
						if ( pBoard[k] == EMTY_FIELD )
						{
							pBoard[k] = actualPlayer;
							pMoves[move] = k;
							lastMove = k;
							goto weiter;
						}
				}
				for ( int j = 0; j < boardFieldCountPlain; j++ )
					for ( int k = j; k < boardFieldCount; k+=boardFieldCountPlain )
						if ( pBoard[k] == EMTY_FIELD) 
						{
							pBoard[k] = actualPlayer;
							pMoves[move] = k;
							lastMove = k;
							goto weiter;
						}
				weiter:;
			}
			else
			{  // normal move
				int bestMove = getBestMove(pBoard, actualPlayer);
				pBoard[bestMove] = actualPlayer;
				pMoves[move] = bestMove;
				lastMove = bestMove;
			}
						
			if ( move == boardFieldCount - 1) 
			{
				// draw
				//draw = true;
				trainValue(pSymmetry->getSymmetry(pBoard), actualPlayer, 0.5, learnRate, maxError);
				pBoard[lastMove]=EMTY_FIELD;
				trainValue(pSymmetry->getSymmetry(pBoard), lastPlayer, 0.5, learnRate, maxError);
				pBoard[pMoves[move-1]]=EMTY_FIELD;
				double valueWinBef = propagate(pSymmetry->getSymmetry(pBoard), actualPlayer);
				trainValue(pSymmetry->getSymmetry(pBoard), actualPlayer, valueWinBef + ( 0.5 - valueWinBef) * alpha, learnRate, maxError);
				pBoard[pMoves[move-2]]=EMTY_FIELD;
				double valueLoseBef = propagate(pSymmetry->getSymmetry(pBoard), lastPlayer);
				trainValue(pSymmetry->getSymmetry(pBoard), lastPlayer, valueLoseBef + ( 0.5 - valueLoseBef) * alpha, learnRate, maxError);
				
				drawCount++;
				break;
			}

			if ( actualPlayer == PLAYER_1)
			{
				//if ( testWinPlayer1.testWin() ) break;
				lastPlayer = PLAYER_1;
				actualPlayer = PLAYER_2;
			} else 
			{
				//if ( testWinPlayer2.testWin() ) break;
				lastPlayer = PLAYER_2;
				actualPlayer = PLAYER_1;
			}

		} // play one complete game loop

		wonGame:;

		if ( maxDepthLocal < move ) maxDepthLocal = move;
		
		if ( pMuele->pTimer->getRealTime() - actualTime > 1000 )
		{
			trainingTime+=pMuele->pTimer->getRealTime() - actualTime;
			actualCycle--;
			break;
		}

		if ( actualCycle % 200  == 0 )
		{
			if ( !pMuele->pNeuralNetworkWindow->pCheckStopAutomatically->isChecked() ) goto skipFurtherTests;
			cout<<" strength test running ";

			int draws1, draws2, playerOneWins1, playerOneWins2, playerTwoWins1, playerTwoWins2;
			int sumdraws1 = 0, sumdraws2 = 0, sumplayerOneWins1 = 0;
			int sumplayerOneWins2 = 0, sumplayerTwoWins1 = 0, sumplayerTwoWins2 = 0;
			
			for ( int i = 0; i < 200; i++ )
			{
				aICompetition( PLAYER_1, 50, playerOneWins1, draws1, playerTwoWins1 );
				sumplayerOneWins1+=playerOneWins1;
				sumdraws1+=draws1;
				sumplayerTwoWins1+=playerTwoWins1;
				if ( sumplayerOneWins1 != 0 )
				{
					cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
					cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
					cout<<" tests: "<<i+1<<"/"<<"200"<<endl; 
					strengthCount=0;
					goto skipFurtherTests;
				}
				aICompetition( PLAYER_2, 50, playerOneWins2, draws2, playerTwoWins2 );
				sumplayerOneWins2+=playerOneWins2;
				sumdraws2+=draws2;
				sumplayerTwoWins2+=playerTwoWins2;
				if ( sumplayerTwoWins2 != 0 )
				{
					cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
					cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
					cout<<" tests: "<<i+1<<"/"<<"200"<<endl; 
					strengthCount=0;
					goto skipFurtherTests;
				}
				else
				if ( strengthCount < 8 )
					{
						strengthCount++;
						cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
						cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
						cout<<" test: "<<strengthCount<<endl;
						goto skipFurtherTests;
					}
			}
			cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
			cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
			cout<<" tests: 200/200"<<endl<<endl; 
			cout<<"Stopped training because of good strength."<<endl<<endl;
			pMuele->trainingRunning=false;
			break;
		}


	skipFurtherTests:;
	}

	if ( maxDepthLocal > maxDepth ) maxDepthCount++; else maxDepthCount=0;
	if ( maxDepthCount > 5 ) 
	{
		maxDepth+=(maxDepthLocal-maxDepth)/3;
		maxDepthCount=0;
	}

	pMuele->gameFinished=true;

	if ( pMuele->trainingRunning )
	{
		memset(pMuele->pBoard, 0, boardFieldCount );
		pMuele->actualPlayer = move % 2 + 1;
		for ( int i = 0; i <= move; i++ )
			pMuele->pBoard[pMoves[i]] = i % 2 + 1;

		cout<<"training cycle: "<<gamesCount-actualCycle<<"/"<<gamesCount<<"  p1/d/p2: "<<win1Count<<"/"<<drawCount<<"/"<<win2Count<<"  max.depth: "<<maxDepth<<endl;
		if ( (gamesCount - actualCycle) < 1000 ) // check if the network output is strange during the first 1000 training cycles
		{
			memset( pBoard, 0, boardFieldCount );
			pBoard[0] = PLAYER_1;
			double prop = propagate(pBoard, PLAYER_1);
			if ( prop > 0.99999 || prop < 0.00001 )
			{
				cout<<endl<<"Bad network output for player 1, training stopped."<<endl;
				resetWeights(PLAYER_1);
				pMuele->trainingRunning=false;
			}
			pBoard[0] = PLAYER_2;
			prop = propagate(pBoard, PLAYER_2);
			if ( prop > 0.99999 || prop < 0.00001 )
			{
				cout<<endl<<"Bad network output for player 2, training stopped."<<endl;
				resetWeights(PLAYER_2);
				pMuele->trainingRunning=false;
			}
		}
	}

	if ( actualCycle <= 0 ) pMuele->trainingRunning = false;

	delete[] pMoves;
	//delete[] pBoard;
}


/********************************************************************/
/* test both players, train the looser, training after complete game*/

void CNeuralNetwork::learnTest()
{

	register char *pMoves  = new char[boardFieldCount];

	char actualPlayer, lastPlayer;    	
	int move;

	double positiveReward = 0.8;
	double negativeReward = 0.2;
	double maxError, learnRate;

	double alpha; // stepsiize param.

	wasTrained = true;

	int cyclesPlayer1 = 0, cyclesPlayer2 = 0, cyclesBoth = 0;
	
	u32 actualTime = pMuele->pTimer->getRealTime();


	int lastMove;
	
	char trainWhichPlayer = 0;
	char trainRandom = 0;

	bool winResult;
	bool trainBothPlayers = false;

	int winCheckFrequenzy = 2500;


	for ( ; actualCycle > 0; actualCycle--)
	{
		maxError = (double)actualCycle / gamesCount * ( maxErrorStart - maxErrorEnd ) + maxErrorEnd;
		learnRate = (double)actualCycle / gamesCount * ( trainRateStart - trainRateEnd ) + trainRateEnd;
		alpha = (double)actualCycle / gamesCount * ( stepSizeStart - stepSizeEnd ) + stepSizeEnd;

		memset(pBoard, 0, boardFieldCount );
		actualPlayer = PLAYER_1;

		// play one complete game
		for ( move = 0;  ; move++ )
		{
			// If actual player can win, take the winning move.
			for ( int j = 0; j < boardFieldCountPlain; j++ ) // number of fields in a plain
			{
				for ( int k = j; k < boardFieldCount; k+=boardFieldCountPlain ) // first free plain
				{
					if ( pBoard[k] == EMTY_FIELD )
					{
						pBoard[k] = actualPlayer;
						if ( actualPlayer == PLAYER_1 )
							winResult = pTestWinPlayer1->testWin();
						else
							winResult = pTestWinPlayer2->testWin();

						if ( winResult ) // one player won the game
						{
							pMoves[move] = k;
							lastMove = k;
                            if ( trainWhichPlayer == 0 ) // set to train the looser
							{

								if ( actualPlayer == PLAYER_1 )
								{
									trainWhichPlayer = PLAYER_2;
									cyclesPlayer2++;
								}
								else
								{
									trainWhichPlayer = PLAYER_1;
									cyclesPlayer1++;
								}
								maxDepth = move;
								goto wonGame;                    
							}
							
							if ( actualPlayer == PLAYER_1 ) win1Count++; else win2Count++;

							// train the player set in trainWhichPlayer

							double valueWinBef = positiveReward;
							double valueLoseBef = negativeReward;

							if ( trainWhichPlayer == actualPlayer || trainBothPlayers )
								trainValue(pSymmetry->getSymmetry(pBoard), actualPlayer, valueWinBef, learnRate, maxError);

							pBoard[pMoves[move]]=EMTY_FIELD;
							
							if ( trainWhichPlayer == lastPlayer || trainBothPlayers )
								trainValue(pSymmetry->getSymmetry(pBoard), lastPlayer, valueLoseBef, learnRate, maxError);
							
							double valueWinActual, valueLoseActual;
							int l = move - 1;
							while ( l >= 0 )
							{
							
								pBoard[pMoves[l]]=EMTY_FIELD;
								if ( trainWhichPlayer == actualPlayer || trainBothPlayers )
								{
									valueWinActual = propagate(pSymmetry->getSymmetry(pBoard), actualPlayer);
									valueWinBef = valueWinActual + ( valueWinBef - valueWinActual) * alpha;

									trainValue(pSymmetry->getSymmetry(pBoard), actualPlayer, valueWinBef, learnRate, maxError);
								}
								l--;
								if ( l < 0 ) goto wonGame;

								pBoard[pMoves[l]]=EMTY_FIELD;
								if ( trainWhichPlayer == lastPlayer || trainBothPlayers )
								{
									valueLoseActual = propagate(pSymmetry->getSymmetry(pBoard), lastPlayer);
									valueLoseBef = valueLoseActual + ( valueLoseBef - valueLoseActual) * alpha;

									trainValue(pSymmetry->getSymmetry(pBoard), lastPlayer, valueLoseBef, learnRate, maxError);
								}
								l--;
								if ( l < 0 ) goto wonGame;
							}

							goto wonGame;
						}
						pBoard[k] = EMTY_FIELD;
						break;
					}
				}
			}
			
//			 make a random move or the best move
//			if ( ( trainWhichPlayer == actualPlayer || trainBothPlayers  ) && (rand() % ( maxDepth / 3  ) == 0) )
//			if ( ( trainWhichPlayer == actualPlayer || trainBothPlayers  ) && (rand() % ((boardFieldCount - move)*2) == 0) )
//			if ( ( trainWhichPlayer == actualPlayer || trainBothPlayers  ) && (rand() % (maxDepth ) == 0) )
			if ( ( trainWhichPlayer == actualPlayer || trainBothPlayers  ) && (rand() % (boardFieldCount) == 0) )
			{  
				//random_move:;
				// exploring move
				int randomField;
				for ( int j = 0; j < boardFieldCountPlain; j++ )
				{
					//randomField =(rand()%(boardFieldCountPlain));
					randomField =(rand()%(boardFieldCountPlain));
					for ( int k = randomField; k < boardFieldCount; k+=boardFieldCountPlain )
						if ( pBoard[k] == EMTY_FIELD )
						{
							pBoard[k] = actualPlayer;
							pMoves[move] = k;
							lastMove = k;
							goto weiter;
						}
				}
				for ( int j = 0; j < boardFieldCountPlain; j++ )
					for ( int k = j; k < boardFieldCount; k+=boardFieldCountPlain )
						if ( pBoard[k] == EMTY_FIELD) 
						{
							pBoard[k] = actualPlayer;
							pMoves[move] = k;
							lastMove = k;
							goto weiter;
						}
				weiter:;
			}
			else
			{  // normal move
				int bestMove = getBestMove(pBoard, actualPlayer);
				pBoard[bestMove] = actualPlayer;
				
				// help player2 sometimes to prevent a winning move of player1
				if ( actualPlayer == PLAYER_2 && move < boardFieldCount - 1 && oldCyclesPlayer1 == 0 
					&& oldCyclesPlayer2 != 0 && (rand() % 2 == 0) )
				{
					int bestMoveNew = getBestMove(pBoard, PLAYER_1);
					// only continue if the move of player1 wouldn't on top of player2's move
					if ( bestMoveNew != bestMove + boardFieldCountPlain )
					{
						pBoard[bestMoveNew] = PLAYER_1;
						if ( pTestWinPlayer1->testWin()  )
						{
							pBoard[bestMove] = EMTY_FIELD;
							bestMove=bestMoveNew;
							pBoard[bestMove] = actualPlayer;
							//goto random_move;
						}
						else
							pBoard[bestMoveNew] = EMTY_FIELD;
					}
					
				}

				pMoves[move] = bestMove;
				lastMove = bestMove;
			}
						
			if ( move == boardFieldCount - 1) 
			{

				double valueWinBef = 0.5;
				double valueLoseBef = 0.5;

				//if ( trainWhichPlayer == actualPlayer || trainBothPlayers )
					trainValue(pSymmetry->getSymmetry(pBoard), actualPlayer, valueWinBef, learnRate, maxError);

				pBoard[pMoves[move]]=EMTY_FIELD;
				
				//if ( trainWhichPlayer == lastPlayer || trainBothPlayers )
					trainValue(pSymmetry->getSymmetry(pBoard), lastPlayer, valueLoseBef, learnRate, maxError);
				
				double valueWinActual, valueLoseActual;
				int l = move - 1;
				while ( l >= 0 )
				{
					pBoard[pMoves[l]]=EMTY_FIELD;
                    valueWinActual = propagate(pSymmetry->getSymmetry(pBoard), actualPlayer);
					valueWinBef = valueWinActual + ( valueWinBef - valueWinActual) * alpha;
					
					//if ( trainWhichPlayer == actualPlayer || trainBothPlayers )
						trainValue(pSymmetry->getSymmetry(pBoard), actualPlayer, valueWinBef, learnRate, maxError);
					l--;
					if ( l < 0 ) break;

					pBoard[pMoves[l]]=EMTY_FIELD;
					valueLoseActual = propagate(pSymmetry->getSymmetry(pBoard), lastPlayer);
					valueLoseBef = valueLoseActual + ( valueLoseBef - valueLoseActual) * alpha;
				
					//if ( trainWhichPlayer == lastPlayer || trainBothPlayers )
						trainValue(pSymmetry->getSymmetry(pBoard), lastPlayer, valueLoseBef, learnRate, maxError);
					l--;
					if ( l < 0 ) break;
				}


				drawCount++;
				break;
			}

			if ( actualPlayer == PLAYER_1)
			{
				lastPlayer = PLAYER_1;
				actualPlayer = PLAYER_2;
			} else 
			{
				lastPlayer = PLAYER_2;
				actualPlayer = PLAYER_1;
			}

		} // play one complete game loop

		wonGame:;

		
		if ( actualCycle % ( boardFieldCount / 2  ) == 0 )
		{
			// reset the player to be trained the next times
			trainWhichPlayer = 0;
			trainRandom = 0;
			trainBothPlayers=false;
			//trainBothPlayers=true;
		}
		else
		{
			if ( trainWhichPlayer == 0 ) 
			{
				trainBothPlayers=true;
				cyclesBoth++;
				trainWhichPlayer=-1;
				maxDepth = boardFieldCount - 1;
				if (oldCyclesPlayer1 == 0 && oldCyclesPlayer2==0) winCheckFrequenzy = 100; else winCheckFrequenzy = 2500;
			}
			
		}

		/*if ( trainRandom ) 
		{ 
			if ( rand() % 2 == 0 ) trainWhichPlayer = PLAYER_1; else trainWhichPlayer = PLAYER_2;
		}
		*/
		if ( pMuele->pTimer->getRealTime() - actualTime > 1000 )
		{
			trainingTime+=pMuele->pTimer->getRealTime() - actualTime;
			actualCycle--;
			break;
		}

		if ( actualCycle % winCheckFrequenzy  == 0 )
		{
			if ( !pMuele->pNeuralNetworkWindow->pCheckStopAutomatically->isChecked() ) goto skipFurtherTests;
			cout<<" strength test running ";

			int draws1, draws2, playerOneWins1, playerOneWins2, playerTwoWins1, playerTwoWins2;
			int sumdraws1 = 0, sumdraws2 = 0, sumplayerOneWins1 = 0;
			int sumplayerOneWins2 = 0, sumplayerTwoWins1 = 0, sumplayerTwoWins2 = 0;
			
			for ( int i = 0; i < 200; i++ )
			{
				aICompetition( PLAYER_1, 50, playerOneWins1, draws1, playerTwoWins1 );
				sumplayerOneWins1+=playerOneWins1;
				sumdraws1+=draws1;
				sumplayerTwoWins1+=playerTwoWins1;
				if ( sumplayerOneWins1 != 0 )
				{
					cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
					
					if ( i == 0 )
					{
						cout<<" skip."<<endl;
						// reset check frequenzy
						winCheckFrequenzy = 2500;
					}
					else 
					{
						cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
						cout<<" tests: "<<i+1<<"/"<<"200"<<endl; 
					}
					strengthCount=0;
					goto skipFurtherTests;
				}
				aICompetition( PLAYER_2, 50, playerOneWins2, draws2, playerTwoWins2 );
				sumplayerOneWins2+=playerOneWins2;
				sumdraws2+=draws2;
				sumplayerTwoWins2+=playerTwoWins2;
				if ( sumplayerTwoWins2 != 0 )
				{
					cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
					cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
					cout<<" tests: "<<i+1<<"/"<<"200"<<endl; 
					strengthCount=0;
					goto skipFurtherTests;
				}
				else
				if ( strengthCount < 8 )
					{
						strengthCount++;
						cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
						cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
						cout<<" test: "<<strengthCount<<endl;
						goto skipFurtherTests;
					}
			}
			cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
			cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
			cout<<" tests: 200/200"<<endl<<endl; 
			cout<<"Stopped training because of good strength."<<endl<<endl;
			pMuele->trainingRunning=false;
			break;
		}

	skipFurtherTests:;
	}

	pMuele->gameFinished=true;

	oldCyclesPlayer1 = cyclesPlayer1;
	oldCyclesPlayer2 = cyclesPlayer2;

	if ( pMuele->trainingRunning )
	{
		memset(pMuele->pBoard, 0, boardFieldCount );
		pMuele->actualPlayer = move % 2 + 1;
		for ( int i = 0; i <= move; i++ )
			pMuele->pBoard[pMoves[i]] = i % 2 + 1;

		cout<<gamesCount-actualCycle<<"/"<<gamesCount<<"  p1/d/p2: "<<win1Count<<"/"<<drawCount
			<<"/"<<win2Count<<"  max.depth: "<<maxDepth<<" cycles: "<<cyclesPlayer1<<"/"<<cyclesPlayer2<<"("<<cyclesBoth<<")"<<endl;
		if ( (gamesCount - actualCycle) < 1000 )
		{
			memset( pBoard, 0, boardFieldCount );
			pBoard[0] = PLAYER_1;
			double prop = propagate(pBoard, PLAYER_1);
			if ( prop > 0.99999 || prop < 0.00001 )
			{
				cout<<endl<<"Bad network output for player 1, training stopped."<<endl;
				resetWeights(PLAYER_1);
				pMuele->trainingRunning=false;
			}
			pBoard[0] = PLAYER_2;
			prop = propagate(pBoard, PLAYER_2);
			if ( prop > 0.99999 || prop < 0.00001 )
			{
				cout<<endl<<"Bad network output for player 2, training stopped."<<endl;
				resetWeights(PLAYER_2);
				pMuele->trainingRunning=false;
			}
		}
	}

	if ( actualCycle <= 0 ) pMuele->trainingRunning = false;

	delete[] pMoves;
}

/********************************************************************/
/* new test */

void CNeuralNetwork::learnTest2()
{

	char *pMoves  = new char[boardFieldCount];
	double *pStateValues = new double[boardFieldCount];


	char actualPlayer, lastPlayer;    	
	int move;

	double positiveReward = 0.8;
	double negativeReward = 0.2;
	double maxError, learnRate;

	double alpha;

	wasTrained = true;

	int cyclesPlayer1 = 0, cyclesPlayer2 = 0;
	
	u32 actualTime = pMuele->pTimer->getRealTime();

	int lastMove;
	
	char trainWhichPlayer = 0;
	char trainRandom = 0;

	bool winResult;
	bool trainBothPlayers = false;

	int winCheckFrequenzy = 2000;

	for ( ; actualCycle > 0; actualCycle--)
	{
		maxError = (double)actualCycle / gamesCount * ( maxErrorStart - maxErrorEnd ) + maxErrorEnd;
		learnRate = (double)actualCycle / gamesCount * ( trainRateStart - trainRateEnd ) + trainRateEnd;
		alpha = (double)actualCycle / gamesCount * ( stepSizeStart - stepSizeEnd ) + stepSizeEnd;

		memset(pBoard, 0, boardFieldCount );
		actualPlayer = PLAYER_1;

		//trainBothPlayers=true;

		// play one complete game
		for ( move = 0;  ; move++ )
		{
			// If actual player can win, take the winning move.
			for ( int j = 0; j < boardFieldCountPlain; j++ )
			{
				for ( int k = j; k < boardFieldCount; k+=boardFieldCountPlain )
				{
					if ( pBoard[k] == EMTY_FIELD )
					{
						pBoard[k] = actualPlayer;
						if ( actualPlayer == PLAYER_1 )
							winResult = pTestWinPlayer1->testWin();
						else
							winResult = pTestWinPlayer2->testWin();

						if ( winResult )
						{
							pMoves[move] = k;
							lastMove = k;
                            if ( trainWhichPlayer == 0 )
							{

								if ( actualPlayer == PLAYER_1 )
								{
									trainWhichPlayer = PLAYER_2;
									cyclesPlayer2++;
								}
								else
								{
									trainWhichPlayer = PLAYER_1;
									cyclesPlayer1++;
								}
								maxDepth = move;
								goto wonGame;
							}
							
							if ( actualPlayer == PLAYER_1 ) win1Count++; else win2Count++;

							int moveSave = move;
							double valueWinBef = positiveReward;
							double valueLoseBef = negativeReward;

							if ( trainWhichPlayer == actualPlayer || trainBothPlayers )
								//trainValue(pSymmetry->getSymmetry(pBoard), actualPlayer, valueWinBef, learnRate, maxError);
								pStateValues[move] = valueWinBef;

							pBoard[pMoves[move]]=EMTY_FIELD;
							
							if ( trainWhichPlayer == lastPlayer || trainBothPlayers )
								//trainValue(pSymmetry->getSymmetry(pBoard), lastPlayer, valueLoseBef, learnRate, maxError);
								pStateValues[move-1] =valueLoseBef;
							
							double valueWinActual, valueLoseActual;
							int l = move - 1;
							while ( l > 0 )
							{
							
								pBoard[pMoves[l]]=EMTY_FIELD;
								if ( trainWhichPlayer == actualPlayer || trainBothPlayers )
								{
									valueWinActual = propagate(pSymmetry->getSymmetry(pBoard), actualPlayer);
									valueWinBef = valueWinActual + ( valueWinBef - valueWinActual) * alpha;

									//trainValue(pSymmetry->getSymmetry(pBoard), actualPlayer, valueWinBef, learnRate, maxError);
									pStateValues[l - 1] = valueWinBef;
								}
								l--;
								if ( l <= 0 ) break;

								pBoard[pMoves[l]]=EMTY_FIELD;
								if ( trainWhichPlayer == lastPlayer || trainBothPlayers )
								{
									valueLoseActual = propagate(pSymmetry->getSymmetry(pBoard), lastPlayer);
									valueLoseBef = valueLoseActual + ( valueLoseBef - valueLoseActual) * alpha;

									//trainValue(pSymmetry->getSymmetry(pBoard), lastPlayer, valueLoseBef, learnRate, maxError);
									pStateValues[l-1] = valueLoseBef;
								}
								l--;
								if ( l <= 0 ) break;
							}

							for ( int o = 0; o < 25; o++ )
							{
								memset(pBoard, 0, boardFieldCount );
								for ( int i = 0; i <= moveSave; i++ )
									pBoard[pMoves[i]] = i % 2 + 1;
                                

								for ( int m = moveSave; m >= 0; m-- )
								{
									if ( m % 2 + 1 == PLAYER_1 )
									{
										if ( trainWhichPlayer == PLAYER_1 || trainBothPlayers )	trainValue(pSymmetry->getSymmetry(pBoard), PLAYER_1, pStateValues[m], learnRate, maxError);
									}
									else
									{
										if ( trainWhichPlayer == PLAYER_2 || trainBothPlayers ) trainValue(pSymmetry->getSymmetry(pBoard), PLAYER_2, pStateValues[m], learnRate, maxError);
									}
									pBoard[pMoves[m]] = EMTY_FIELD;


								}
							}

							goto wonGame;
						}
						pBoard[k] = EMTY_FIELD;
						break;
					}
				}
			}
			
			//if ( maxDepth < move ) maxDepth = move;			

//			 make a random move or the best move
			//if ( mt_random() % ( maxDepth / 2 ) == 0)
			//(mt_random() % ( boardFieldCount / 2  ) == 0)
//			if ( mt_random() % ( 1 + ( boardFieldCount ) - ( move )  ) == 0 )
//			if ( trainWhichPlayer == actualPlayer && (mt_random() % ( boardFieldCount / 2  ) == 0) )
//			int propVal;
//			if ( move < maxDepth ) propVal = maxDepth - move + 1; else propVal = maxDepth / 2;
			if ( ( trainWhichPlayer == actualPlayer || trainBothPlayers  ) && (rand() % ( maxDepth / 2  ) == 0) )
//			if ( ( trainWhichPlayer == actualPlayer || trainBothPlayers  ) && (rand() % propVal == 0) )
//			if ( trainWhichPlayer == actualPlayer && (mt_random() % (  ( maxDepth ) - ( move ) +1  ) == 0) )
			{  
				// exploring move
				int randomField;
				for ( int j = 0; j < boardFieldCountPlain; j++ )
				{
					//randomField =(rand()%(boardFieldCountPlain));
					randomField =(rand()%(boardFieldCountPlain));
					for ( int k = randomField; k < boardFieldCount; k+=boardFieldCountPlain )
						if ( pBoard[k] == EMTY_FIELD )
						{
							pBoard[k] = actualPlayer;
							pMoves[move] = k;
							lastMove = k;
							goto weiter;
						}
				}
				for ( int j = 0; j < boardFieldCountPlain; j++ )
					for ( int k = j; k < boardFieldCount; k+=boardFieldCountPlain )
						if ( pBoard[k] == EMTY_FIELD) 
						{
							pBoard[k] = actualPlayer;
							pMoves[move] = k;
							lastMove = k;
							goto weiter;
						}
				weiter:;
			}
			else
			{  // normal move
				int bestMove = getBestMove(pBoard, actualPlayer);
				pBoard[bestMove] = actualPlayer;
				pMoves[move] = bestMove;
				lastMove = bestMove;
			}
						
			if ( move == boardFieldCount - 1) 
			{

				double valueWinBef = 0.5;
				double valueLoseBef = 0.5;

				int moveSave = move;

				pStateValues[move] = valueWinBef;
				pBoard[pMoves[move]]=EMTY_FIELD;
				pStateValues[move-1] =valueLoseBef;
				
				double valueWinActual, valueLoseActual;
				int l = move - 1;
				while ( l > 0 )
				{
				
					pBoard[pMoves[l]]=EMTY_FIELD;
					valueWinActual = propagate(pSymmetry->getSymmetry(pBoard), actualPlayer);
					valueWinBef = valueWinActual + ( valueWinBef - valueWinActual) * alpha;
					pStateValues[l - 1] = valueWinBef;

					l--;
					if ( l <= 0 ) break;

					pBoard[pMoves[l]]=EMTY_FIELD;
					valueLoseActual = propagate(pSymmetry->getSymmetry(pBoard), lastPlayer);
					valueLoseBef = valueLoseActual + ( valueLoseBef - valueLoseActual) * alpha;
					pStateValues[l-1] = valueLoseBef;

					l--;
					if ( l <= 0 ) break;
				}

				for ( int o = 0; o < 25; o++ )
				{
					memset(pBoard, 0, boardFieldCount );
					for ( int i = 0; i <= moveSave; i++ )
						pBoard[pMoves[i]] = i % 2 + 1;
                    

					for ( int m = moveSave; m >= 0; m-- )
					{
						if ( m % 2 + 1 == PLAYER_1 )
						{
							trainValue(pSymmetry->getSymmetry(pBoard), PLAYER_1, pStateValues[m], learnRate, maxError);
						}
						else
						{
							trainValue(pSymmetry->getSymmetry(pBoard), PLAYER_2, pStateValues[m], learnRate, maxError);
						}
						pBoard[pMoves[m]] = EMTY_FIELD;
					}
				}

				drawCount++;
				break;
			}

			if ( actualPlayer == PLAYER_1)
			{
				lastPlayer = PLAYER_1;
				actualPlayer = PLAYER_2;
			} else 
			{
				lastPlayer = PLAYER_2;
				actualPlayer = PLAYER_1;
			}

		} // play one complete game loop

		wonGame:;



		if ( actualCycle % ( boardFieldCount * 2  ) == 0 )
		{
			trainWhichPlayer = 0;
			trainRandom = 0;
			trainBothPlayers=false;
		}
		else
		{
			if ( trainWhichPlayer == 0 ) 
			{
				trainBothPlayers=true;
				trainWhichPlayer=-1;
				maxDepth = boardFieldCount - 1;
				//trainRandom = 1;
				if (cyclesPlayer1 == 0 && cyclesPlayer2==0) winCheckFrequenzy = 100; else winCheckFrequenzy = 5000;
			}
			
		}

		/*if ( trainRandom ) 
		{ 
			if ( rand() % 2 == 0 ) trainWhichPlayer = PLAYER_1; else trainWhichPlayer = PLAYER_2;
		}
		*/
		if ( pMuele->pTimer->getRealTime() - actualTime > 1000 )
		{
			trainingTime+=pMuele->pTimer->getRealTime() - actualTime;
			actualCycle--;
			break;
		}

		if ( actualCycle % winCheckFrequenzy  == 0 )
		{
			if ( !pMuele->pNeuralNetworkWindow->pCheckStopAutomatically->isChecked() ) goto skipFurtherTests;
			cout<<" strength test running ";

			int draws1, draws2, playerOneWins1, playerOneWins2, playerTwoWins1, playerTwoWins2;
			int sumdraws1 = 0, sumdraws2 = 0, sumplayerOneWins1 = 0;
			int sumplayerOneWins2 = 0, sumplayerTwoWins1 = 0, sumplayerTwoWins2 = 0;
			
			for ( int i = 0; i < 200; i++ )
			{
				aICompetition( PLAYER_1, 50, playerOneWins1, draws1, playerTwoWins1 );
				sumplayerOneWins1+=playerOneWins1;
				sumdraws1+=draws1;
				sumplayerTwoWins1+=playerTwoWins1;
				if ( sumplayerOneWins1 != 0 )
				{
					cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
					
					if ( i == 0 ) cout<<" skip."<<endl;
					else 
					{
						cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
						cout<<" tests: "<<i+1<<"/"<<"200"<<endl; 
					}
					strengthCount=0;
					goto skipFurtherTests;
				}
				aICompetition( PLAYER_2, 50, playerOneWins2, draws2, playerTwoWins2 );
				sumplayerOneWins2+=playerOneWins2;
				sumdraws2+=draws2;
				sumplayerTwoWins2+=playerTwoWins2;
				if ( sumplayerTwoWins2 != 0 )
				{
					cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
					cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
					cout<<" tests: "<<i+1<<"/"<<"200"<<endl; 
					strengthCount=0;
					goto skipFurtherTests;
				}
				else
				if ( strengthCount < 8 )
					{
						strengthCount++;
						cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
						cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
						cout<<" test: "<<strengthCount<<endl;
						goto skipFurtherTests;
					}
			}
			cout<<"player1: "<<sumplayerOneWins1<<"/"<<sumdraws1<<"/"<<sumplayerTwoWins1;
			cout<<" player2: "<<sumplayerOneWins2<<"/"<<sumdraws2<<"/"<<sumplayerTwoWins2;
			cout<<" tests: 200/200"<<endl<<endl; 
			cout<<"Stopped training because of good strength."<<endl<<endl;
			pMuele->trainingRunning=false;
			break;
		}

	skipFurtherTests:;
	}

	pMuele->gameFinished=true;

	if ( pMuele->trainingRunning )
	{
		memset(pMuele->pBoard, 0, boardFieldCount );
		pMuele->actualPlayer = move % 2 + 1;
		for ( int i = 0; i <= move; i++ )
			pMuele->pBoard[pMoves[i]] = i % 2 + 1;

		cout<<gamesCount-actualCycle<<"/"<<gamesCount<<"  p1/d/p2: "<<win1Count<<"/"<<drawCount<<"/"<<win2Count<<"  max.depth: "<<maxDepth<<" cycles: "<<cyclesPlayer1<<"/"<<cyclesPlayer2<<endl;
		if ( (gamesCount - actualCycle) < 1000 )
		{
			memset( pBoard, 0, boardFieldCount );
			pBoard[0] = PLAYER_1;
			double prop = propagate(pBoard, PLAYER_1);
			if ( prop > 0.99999 || prop < 0.00001 )
			{
				cout<<endl<<"Bad network output for player 1, training stopped."<<endl;
				resetWeights(PLAYER_1);
				pMuele->trainingRunning=false;
			}
			pBoard[0] = PLAYER_2;
			prop = propagate(pBoard, PLAYER_2);
			if ( prop > 0.99999 || prop < 0.00001 )
			{
				cout<<endl<<"Bad network output for player 2, training stopped."<<endl;
				resetWeights(PLAYER_2);
				pMuele->trainingRunning=false;
			}
		}
	}

	if ( actualCycle <= 0 ) pMuele->trainingRunning = false;

    

	//cout<<"win1: "<<win1Count<<" draws: "<<drawCount<<" win2: "<<win2Count<<endl;

	delete[] pMoves;
	delete[] pStateValues;
}

/********************************************************************/

void CNeuralNetwork::xorTest()
{
	double onVal = 0.9;
	double offVal = 0.1;

	cout<<"XOR Test:"<<endl<<endl;

	for ( int i = 0; i < 75; i++ )
	{
		pBoard[0] = 0; pBoard[1] = 0;
		trainValue(pBoard, PLAYER_1, offVal, 0.2, 0.000002);
		pBoard[0] = 0; pBoard[1] = 1;
		trainValue(pBoard, PLAYER_1, onVal, 0.2, 0.000002);
		pBoard[0] = 1; pBoard[1] = 0;
		trainValue(pBoard, PLAYER_1, onVal, 0.2, 0.000002);
		pBoard[0] = 1; pBoard[1] = 1;
		trainValue(pBoard, PLAYER_1, offVal, 0.2, 0.000002);
	}


	pBoard[0] = 0; pBoard[1] = 0;
	cout<<"00: "<<propagate(pBoard, PLAYER_1, true);
	if ( fabs(propagate(pBoard, PLAYER_1, true) - offVal ) < 0.001 ) cout<<" OK"<<endl; else cout<<" NOK"<<endl;
	pBoard[0] = 0; pBoard[1] = 1;
	cout<<"01: "<<propagate(pBoard, PLAYER_1, true);
	if ( fabs(propagate(pBoard, PLAYER_1, true) - onVal ) < 0.001 ) cout<<" OK"<<endl; else cout<<" NOK"<<endl;
	pBoard[0] = 1; pBoard[1] = 0;
	cout<<"10: "<<propagate(pBoard, PLAYER_1, true);
	if ( fabs(propagate(pBoard, PLAYER_1, true) - onVal ) < 0.001 ) cout<<" OK"<<endl; else cout<<" NOK"<<endl;
	pBoard[0] = 1; pBoard[1] = 1;
	cout<<"11: "<<propagate(pBoard, PLAYER_1, true);
	if ( fabs(propagate(pBoard, PLAYER_1, true) - offVal ) < 0.001 ) cout<<" OK"<<endl; else cout<<" NOK"<<endl;

	cout<<endl<<"Enter something and press enter."<<endl;
	char a;
	std::cin>>a;

	exit(0);
}
