Подсчет строк в гигантском текстовом файле > 4 Гб

#include "stdafx.h"
#include <iostream>
using std::cin;
using std::cout;
using std::endl;
using std::cerr;
#include <string>
using std::string;
#include <fstream>
using std::ifstream;
#include <Windows.h>
 
 
string man = "This is programm to count number of lines in text file.\nTo use program type:\ncounter.exe \"filename\"\n\n";
static int numlines=1;
DWORD BUFFERSIZE = 1048576;//buffer size for memory mapping. 1 Mb by default. You can change it to play with performance.
HANDLE hMapFile;      // handle for the file's memory-mapped region
HANDLE hFile;         // the file handle
DWORD dwFileSize;     // temporary storage for file sizes
LPVOID lpMapAddress;  // pointer to the base address of the memory-mapped region
SYSTEM_INFO SysInfo;  // system information; used to get granularity
DWORD dwSysGran;      // system allocation granularity
char * pData;         // pointer to the data
DWORD dwi=0;          // loop counter
BOOL bFlag;           // a result holder
DWORD dwMapViewSize;  // number of bytes to map
DWORD dwCurrentPosition; //position where to start mapping of the file
DWORD dwIterationTimes; //number of iterations to mat the file to the memory
DWORD dwFileMapSize;   // size of the file mapping
 
 
 
int _tmain(int argc, _TCHAR* argv[])
{
    //we accept only one file name
    if (argc != 2 ){cout << man; 
    exit(1);}
     _tprintf(TEXT("Open file \"%s\" for line counting.\n"), argv[1]);
 
// Open the file 
  hFile = CreateFile(argv[1],
                     GENERIC_READ,
                     0,
                     NULL,
                     OPEN_EXISTING,
                     FILE_ATTRIBUTE_NORMAL,
                     NULL);
 
  if (hFile == INVALID_HANDLE_VALUE)
  {_tprintf(TEXT("Target file \"%s\" does not exist. Try to check spelling.\n"), argv[1]);
    exit(1);};
 
 
    //Now we will count some variables ************************************
    //Gety file size.
    dwFileSize = GetFileSize(hFile,  NULL);
    _tprintf(TEXT("File size is: %3d bites\n"), dwFileSize); 
 
    // Get the system allocation granularity.
    GetSystemInfo(&SysInfo);
    dwSysGran = SysInfo.dwAllocationGranularity;
    //_tprintf(TEXT("system allocation granularity is: %d\n"), dwSysGran);
 
    //Calculate number iterations to read all file maps
    
    if (dwFileSize <= BUFFERSIZE) {dwIterationTimes = 1;};
            if (dwFileSize > BUFFERSIZE) {
                DWORD dba = dwFileSize % BUFFERSIZE ;
                if ( dba == 0)  {dwIterationTimes = dwFileSize / BUFFERSIZE;};
                if (dba > 0)  {dwIterationTimes = (dwFileSize / BUFFERSIZE)+1;};
            };
        //_tprintf(TEXT("Iterations: %3d\n"), dwIterationTimes);
 
 
        // now we count some changing variables for cycle
        for  (DWORD c=0 ; c < dwIterationTimes; c=c+1 ) {
                //Count file mapping size and wiev size size for NOT last itteration
                if (c < dwIterationTimes-1){
                    dwMapViewSize=BUFFERSIZE;
                    dwFileMapSize = (c * BUFFERSIZE)  + BUFFERSIZE; //file mapping size
                    //_tprintf (TEXT("The middle file mapping object is %ld bytes large for %ld itteration\n"),dwFileMapSize, c+1);
                    };
 
                ///Count file mapping size and wiev size for last itteration itteration
                if (c == dwIterationTimes-1){
                    DWORD d = (dwFileSize - (c * BUFFERSIZE));//length of last part (less than buffersize)
                    //_tprintf (TEXT("Current offset is %ld, Length of last part of file is %ld \n"),(c * BUFFERSIZE), d);
                    dwMapViewSize = d;
                    dwFileMapSize = dwFileSize;
                    //_tprintf (TEXT("The LAST file mapping object is %ld bytes large for %ld itteration\n"),dwFileMapSize, c+1);
                };//if it is last operation we do not need to allocate all buffer size
        
        
                // Create a file mapping object for the file
                hMapFile = CreateFileMapping( hFile,          // current file handle
                    NULL,           // default security
                    PAGE_READONLY,  // read permission
                    0,              // size of mapping object, high
                    dwFileMapSize,  // size of mapping object, low
                    NULL);          // name of mapping object
                //That it is a good idea to ensure the file size is not zero
                if (hMapFile == NULL)
                    {
                    _tprintf(TEXT("hMapFile is NULL\n"));
                    exit(2);
                    };
 
                // Map the view and test the results.       
                lpMapAddress = MapViewOfFile(hMapFile,            // handle to mapping object
                               FILE_MAP_READ,       // read/write
                               0,                   // high-order 32 bits of file offset
                               (c * BUFFERSIZE),    // low-order 32 bits of file offset
                               dwMapViewSize);      
                if (lpMapAddress == NULL)
                {
                _tprintf(TEXT("lpMapAddress is NULL: last error: %d\n"), GetLastError());
                exit(1);
                }
                else {
                //  _tprintf(TEXT("lpMapAddress is: %d\n"), lpMapAddress);
                };
 
 
                //pointer to the data.
                pData = (char *) lpMapAddress ;
 
                // At last!!! Data processing.************************************************* 
  
                for (dwi=0; dwi < dwMapViewSize ; dwi=dwi+1) {
                if (pData[dwi] == '\n'){numlines++;};
                };
                
 
                // Close the file mapping object and the open file
                 bFlag = UnmapViewOfFile(lpMapAddress);
                 bFlag = CloseHandle(hMapFile); // close the file mapping object
                 if(!bFlag) { _tprintf(TEXT("\nError %ld occurred closing the mapping object!"), GetLastError());}
 
        };//end of for
 
        bFlag = CloseHandle(hFile);   // close the file itself
        if(!bFlag) { _tprintf(TEXT("\nError %ld occurred closing the file!"), GetLastError());}
        
        
        cout << "Number of lines in our file is: " << numlines << endl;
    return 0;
}

hMapFile = CreateFileMapping( hFile,          // current file handle
                    NULL,           // default security
                    PAGE_READONLY,  // read permission
                    ??????,              // size of mapping object, high
                    ??????, // size of mapping object, low
                    NULL);          // name of mapping object
 
lpMapAddress = MapViewOfFile(hMapFile,            // handle to mapping object
                               FILE_MAP_READ,       // read/write
                               ??????,                   // high-order 32 bits of file offset
                               ??????,  // low-order 32 bits of file offset
                               dwMapViewSize);

#include <iostream>
#include <windows.h>
 
int main()
{
    __int64 x = 42949672960;
    DWORD low = (DWORD)x;
    DWORD high = (DWORD)(x>>32);
 
    std::cout<<x<<std::endl;
    std::cout<<low<<std::endl;
    std::cout<<high<<std::endl;
 
 
    return 0;
}

#include "stdafx.h"
#include <iostream>
using std::cin;
using std::cout;
using std::endl;
using std::cerr;
#include <string>
using std::string;
#include <Windows.h>
#include <time.h>
 
 
string man = "This is programm to count number of lines in text file.\nTo use program type:\ncounter.exe \"filename\"\n\n";
static int numlines=1;
LARGE_INTEGER liBufferSize; //buffer size for memory mapping. 1 Mb by default. You can change it to play with performance.
HANDLE hMapFile;      // handle for the file's memory-mapped region
HANDLE hFile;         // the file handle
LARGE_INTEGER liFileSize; // temporary storage for file sizes for big files
LPVOID lpMapAddress;  // pointer to the base address of the memory-mapped region
SYSTEM_INFO SysInfo;  // system information; used to get granularity
DWORD dwSysGran;      // system allocation granularity
char * pData;         // pointer to the data
int inti=0;          // loop counter for data manipulation
BOOL bFlag;           // a result holder
int intIterationTimes; //number of iterations to mat the file to the memory
LARGE_INTEGER liMapViewSize;  // number of bytes to map
LARGE_INTEGER liOffset; //current position holder
 
 
 
int _tmain(int argc, _TCHAR* argv[])
{
    clock_t tStart = clock();
    liBufferSize.LowPart = 1048576;// 1 Mb by default. You can change it to play with performance.
    
    //we accept only one file name
    if (argc != 2 ){cout << man; 
    exit(1);}
     _tprintf(TEXT("Open file \"%s\" for line counting.\n"), argv[1]);
 
// Open the file 
  hFile = CreateFile(argv[1],
                     GENERIC_READ,
                     0,
                     NULL,
                     OPEN_EXISTING,
                     FILE_ATTRIBUTE_NORMAL,
                     NULL);
 
    if (hFile == INVALID_HANDLE_VALUE)
    {_tprintf(TEXT("Target file \"%s\" does not exist. Try to check spelling.\n"), argv[1]);
    exit(1);
    };
 
 
    //Now we will count some variables ************************************
    //Gety file size.
    GetFileSizeEx(hFile, &liFileSize);
    _tprintf(TEXT("File size is: %I64d bites\n"), liFileSize.QuadPart); 
    
    // Get the system allocation granularity.
    GetSystemInfo(&SysInfo);
    dwSysGran = SysInfo.dwAllocationGranularity;
    _tprintf(TEXT("system allocation granularity is: %d\n"), dwSysGran);
 
    //Calculate number iterations to read all file maps
    if (liFileSize.QuadPart <= liBufferSize.QuadPart) {intIterationTimes = 1;};
    if (liFileSize.QuadPart > liBufferSize.QuadPart) {
                LONGLONG dba = liFileSize.QuadPart % liBufferSize.QuadPart ;
                if ( dba == 0)  {intIterationTimes = (int)(liFileSize.QuadPart / liBufferSize.QuadPart);};
                if (dba > 0)  {intIterationTimes = (int)((liFileSize.QuadPart / liBufferSize.QuadPart)+1);};
    };
    _tprintf(TEXT("Iterations: %u \n"), intIterationTimes);
 
    // Create a file mapping object for the file
    hMapFile = CreateFileMapping( hFile,          // current file handle
            NULL,           // default security
            PAGE_READONLY,  // read permission
            (DWORD)liFileSize.HighPart,  // size of mapping object, high
            liFileSize.LowPart, // size of mapping object, low
            NULL);          // name of mapping object
    //That it is a good idea to ensure the file size is not zero
    if (hMapFile == NULL){
        _tprintf(TEXT("hMapFile is NULL\n"));
        exit(2);
    };
 
 
    // now we count some changing variables for cycle
    for  (int intc=0 ; intc < intIterationTimes; intc++ ) {
        //Count  map wiev size  for NOT last itteration
        if (intc < intIterationTimes-1){
            liMapViewSize.LowPart=liBufferSize.LowPart;
        //  _tprintf (TEXT("The middle file mapping view is %I64d bytes large for %i itteration\n"),liMapViewSize.QuadPart, (intc+1));
            };
 
        ///Count file wiev size and current offset for last itteration 
        if (intc == intIterationTimes-1){
            liMapViewSize.QuadPart = (liFileSize.QuadPart - (   (LONGLONG)intc * liBufferSize.QuadPart)  );//length of last part (less than liBufferSize)
            liOffset.QuadPart= (LONGLONG)intc * liBufferSize.QuadPart;
            _tprintf (TEXT("Current offset is %I64d, Length of last part of view is %I64d \n"),liOffset.QuadPart, liMapViewSize.QuadPart);
            };//if it is last operation we do not need to allocate all buffer size
        
 
        // Map the view and test the results.       
        lpMapAddress = MapViewOfFile(
                hMapFile,               // handle to mapping object
                FILE_MAP_READ,          // read/write
                liOffset.HighPart,      // high-order 32 bits of file offset
                liOffset.LowPart ,      // low-order 32 bits of file offset
                liMapViewSize.LowPart);  //length of the view    
        
        if (lpMapAddress == NULL){
            _tprintf(TEXT("lpMapAddress is NULL: last error: %d\n"), GetLastError());
            exit(1);}
        else {
            //_tprintf(TEXT("lpMapAddress is: %d\n"), lpMapAddress);
        };
 
 
        //pointer to the data.
        pData = (char *) lpMapAddress ;
 
        // At last!!! Data processing.************************************************* 
        for (inti=0; (DWORD)inti < liMapViewSize.LowPart ; inti++) {
                if (pData[inti] == '\n'){numlines++;};
                };
                
 
        // Close the file mapping object and the open file
         bFlag = UnmapViewOfFile(lpMapAddress);
                
    };//end of for
 
    bFlag = CloseHandle(hMapFile); // close the file mapping object
    if(!bFlag)  { _tprintf(TEXT("\nError %ld occurred closing the mapping object!"), GetLastError());}
    bFlag = CloseHandle(hFile);   // close the file itself
    if(!bFlag) { _tprintf(TEXT("\nError %ld occurred closing the file!"), GetLastError());}
        
    //now we cout the reults
    cout << "Number of lines in our file is: " << numlines << endl;
    printf("Time taken: %.2fs\n", (double)(clock() - tStart)/CLOCKS_PER_SEC);
 
    return 0;
}