From 232a59569c10f8ae704497ea2f50fcbfc06f17f3 Mon Sep 17 00:00:00 2001 From: Vladimir K <1472540+vladimir0ne1@users.noreply.github.com> Date: Sun, 23 Jul 2023 17:10:20 +0300 Subject: [PATCH] Initial setup - gitea --- PriorBankParser/Constants.cs | 62 ++++++- PriorBankParser/DbLoader.cs | 61 +++++++ PriorBankParser/MemberMapX.cs | 14 ++ PriorBankParser/Parser.cs | 167 +++++++++++------- PriorBankParser/PriorBankParser.csproj | 16 ++ PriorBankParser/SectionParseResult.cs | 11 -- PriorBankParser/StatementDto.cs | 42 ----- ...ransactionInfoDto.cs => TransactionDto.cs} | 16 +- PriorBankParser/TransactionInfoDtoReadMap.cs | 21 +++ PriorBankParser/TransactionInfoDtoWriteMap.cs | 45 +++++ PriorBankParser/download_receipt.js | 26 +++ TransactionListParser.sln | 16 ++ 12 files changed, 370 insertions(+), 127 deletions(-) create mode 100644 PriorBankParser/DbLoader.cs create mode 100644 PriorBankParser/MemberMapX.cs create mode 100644 PriorBankParser/PriorBankParser.csproj delete mode 100644 PriorBankParser/SectionParseResult.cs delete mode 100644 PriorBankParser/StatementDto.cs rename PriorBankParser/{TransactionInfoDto.cs => TransactionDto.cs} (55%) create mode 100644 PriorBankParser/TransactionInfoDtoReadMap.cs create mode 100644 PriorBankParser/TransactionInfoDtoWriteMap.cs create mode 100644 PriorBankParser/download_receipt.js create mode 100644 TransactionListParser.sln diff --git a/PriorBankParser/Constants.cs b/PriorBankParser/Constants.cs index 913e460..e3c5b18 100644 --- a/PriorBankParser/Constants.cs +++ b/PriorBankParser/Constants.cs @@ -1,11 +1,65 @@ -namespace PriorBankParser +using System.Globalization; +using CsvHelper.Configuration; + +namespace PriorBankParser { - public static class Constants + internal static class Constants { - public static class SectionNames + internal static class SourceConfig + { + internal const string CodePage = "windows-1251"; + + internal const string CultureInfoName = "ru-RU"; + + internal static CultureInfo SrcCultureInfo { get; } = new CultureInfo(CultureInfoName); + + internal static CsvConfiguration SrcCsvConfiguration { get; } = GetReadConfig(); + + internal const string CsvDelimiter = ";"; + } + + private static CsvConfiguration GetReadConfig() + { + var config = new CsvConfiguration(SourceConfig.SrcCultureInfo) + { + Delimiter = SourceConfig.CsvDelimiter, + IgnoreBlankLines = true, + IgnoreQuotes = true, + }; + + config.RegisterClassMap(); + return config; + } + + public static CsvConfiguration GetWriteConfig() + { + var config = new CsvConfiguration(CultureInfo.InvariantCulture) + { + Delimiter = ",", + ShouldQuote = (s, context) => true, + }; + + config.RegisterClassMap(); + return config; + } + + internal static class SourceColumns + { + internal const string TransactionDate = "Дата транзакции"; + internal const string Operation = "Операция"; + internal const string Amount = "Сумма"; + internal const string Currency = "Валюта"; + internal const string OperationDate = "Дата операции по счету"; + internal const string Commission = "Комиссия/Money-back"; + internal const string AccountTurnover = "Обороты по счету"; + internal const string Category = "Категория операции"; + } + + internal static class SectionNames { - internal const string LockedSectionPrefix = "Заблокированные суммы по ........"; internal const string OperationSectionPrefix = "Операции по ........"; + internal const string CardTotalSectionPrefix = "Всего по контракту"; + internal const string StatementTotalSectionPrefix = "Всего в данной валюте"; } } } diff --git a/PriorBankParser/DbLoader.cs b/PriorBankParser/DbLoader.cs new file mode 100644 index 0000000..04c3410 --- /dev/null +++ b/PriorBankParser/DbLoader.cs @@ -0,0 +1,61 @@ +using System.Data.SqlClient; +using System.Linq; +using Dapper; +using NUnit.Framework; + +namespace PriorBankParser +{ + public class DbLoader + { + private const string Sql = @" +DROP TABLE IF EXISTS [dbo].[VPSK_IMPORT] +GO + +CREATE TABLE [dbo].[VPSK_IMPORT] +( + [Contract] NVARCHAR(255), + [TransactionName] NVARCHAR(2048), + [Category] NVARCHAR(2048), + [Currency] VARCHAR(5), + [IsIncome] VARCHAR(5), + [TransactionDate] DATETIME, + [OperationDate] DATETIME, + [Amount] DECIMAL(19, 6), + [Commission] DECIMAL(19, 6), + [AccountTurnover] DECIMAL(19, 6) +) +GO + +BULK INSERT [dbo].[VPSK_IMPORT] +FROM '' +WITH ( + TABLOCK + ,FORMAT = 'CSV' + ,FIRSTROW = 2 + ,FIELDTERMINATOR = ',' + ,FIELDQUOTE = '0x22' + --,ROWTERMINATOR = '0x0a' + ,CODEPAGE=65001 +); + +"; + + [Test] + public void Load() + { + const string sourceFile = @"D:\Cloud-storage\Dropbox\Home\Finance\Raw Source\ConvertedResult.csv"; + var script = Sql.Replace("", sourceFile); + + var connectionString = new SqlConnectionStringBuilder + { DataSource = "(local)", InitialCatalog = "MY_FINANCE", IntegratedSecurity = true, } + .ConnectionString; + using var sqlConnection = new SqlConnection(connectionString); + sqlConnection.Open(); + + foreach (var scriptPart in script.Split("GO").Where(s => !string.IsNullOrEmpty(s))) + { + sqlConnection.Execute(scriptPart, commandTimeout: 60 * 60); + } + } + } +} diff --git a/PriorBankParser/MemberMapX.cs b/PriorBankParser/MemberMapX.cs new file mode 100644 index 0000000..cd6047d --- /dev/null +++ b/PriorBankParser/MemberMapX.cs @@ -0,0 +1,14 @@ +using System.Globalization; +using CsvHelper.Configuration; + +namespace PriorBankParser +{ + public static class MemberMapX + { + public static MemberMap WithDecimalConversion(this MemberMap map) + { + return map.TypeConverterOption.NumberStyles(NumberStyles.Any) + .TypeConverterOption.CultureInfo(Constants.SourceConfig.SrcCultureInfo); + } + } +} \ No newline at end of file diff --git a/PriorBankParser/Parser.cs b/PriorBankParser/Parser.cs index 72fdd11..405a49c 100644 --- a/PriorBankParser/Parser.cs +++ b/PriorBankParser/Parser.cs @@ -1,83 +1,124 @@ -using System; -using System.Collections.Generic; -using System.Data; -using System.Globalization; +using System.Collections.Generic; using System.IO; +using System.Text; using CsvHelper; -using CsvHelper.Configuration; +using NUnit.Framework; +using PriorBankParser.Dtos; +using static PriorBankParser.Constants.SectionNames; +using static PriorBankParser.Constants.SourceConfig; namespace PriorBankParser { - public class SectionParser + public class Parser { - public enum SectionType + [Test] + public void Run() { - AccountInfo, - StatementDetails, - ContractOperations, - ContractLocked, - ContractTotal, - StatementTotal + const string sourceDirectoryPath = @"D:\Cloud-storage\Dropbox\Home\Finance\Raw Source\"; + const string resultFileName = @"ConvertedResult.csv"; + var result = ParseFilesInDirectory(sourceDirectoryPath); + var resultFilePath = Path.Combine(sourceDirectoryPath, resultFileName); + WriteResult(resultFilePath, result); } - public SectionParseResultDto Parse(string filePath) + private ICollection ParseFilesInDirectory(string directoryPath) { - var result = new SectionParseResultDto(); - var transactions = new List(); - result.Transactions = transactions; + var files = Directory.GetFiles(directoryPath, "*.csv"); + var result = new List(); - using var reader = new StreamReader(filePath); - using var csv = new CsvReader(reader, CultureInfo.InvariantCulture); - csv.Configuration.HasHeaderRecord = false; - csv.Configuration.RegisterClassMap(); - //csv.Configuration. - - var currentSection = SectionType.AccountInfo; - var contractName = string.Empty; - - while (csv.Read()) + foreach (var file in files) { - var field = csv.GetField(0); - if (field.Equals(Constants.SectionNames.OperationSectionPrefix)) - { - currentSection = SectionType.StatementTotal; - contractName = field.Replace(Constants.SectionNames.OperationSectionPrefix, string.Empty); - continue; - } - - //switch (currentSection) - //{ - // case "FooId": - // fooRecords.Add(csv.GetRecord()); - // break; - // case "BarId": - // barRecords.Add(csv.GetRecord()); - // break; - // default: - // throw new InvalidOperationException("Unknown record type."); - //} + result.AddRange(ParseFile(file)); } return result; } - - } - - public sealed class TransactionInfoDtoMap : ClassMap - { - public TransactionInfoDtoMap() + private ICollection ParseFile(string filePath) { - Map(m => m.TransactionDate).Name("Дата транзакции"); - Map(m => m.OperationName).Name("Операция"); - Map(m => m.Amount).Name("Сумма"); - Map(m => m.Currency).Name("Валюта"); - Map(m => m.OperationDate).Name("Дата операции по счету"); - Map(m => m.Commission).Name("Комиссия/Money-back"); - Map(m => m.AccountTurnover).Name("Обороты по счету"); - Map(m => m.DigitalCard).Name("Цифровая карта"); - Map(m => m.OperationCategory).Name("Категория операции"); + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + + var result = new List(); + using var reader = new StreamReader(filePath, Encoding.GetEncoding(CodePage)); + + var contractName = SeekToTheTransactions(reader); + + if (string.IsNullOrEmpty(contractName)) + { + return result; + } + + using var csvReader = new CsvReader(reader, SrcCsvConfiguration); + + var isHeader = true; + var ignore = false; + + while (csvReader.Read()) + { + var field = csvReader.GetField(0); + + if (field.Equals(StatementTotalSectionPrefix)) + { + break; + } + + if (field.Equals(CardTotalSectionPrefix)) + { + ignore = true; + } + + if (field.StartsWith(OperationSectionPrefix)) + { + contractName = GetCardName(field); + isHeader = true; + continue; + } + + if (isHeader) + { + csvReader.ReadHeader(); + isHeader = false; + ignore = false; + continue; + } + + if (ignore) + { + continue; + } + + var record = csvReader.GetRecord(); + record.Contract = contractName; + result.Add(record); + } + + return result; + } + + private void WriteResult(string resultFilePath, IEnumerable records) + { + using var writer = new StreamWriter(resultFilePath, false, Encoding.UTF8); + using var csv = new CsvWriter(writer, Constants.GetWriteConfig()); + csv.WriteRecords(records); + } + + private string SeekToTheTransactions(StreamReader reader) + { + while (!reader.EndOfStream) + { + var line = reader.ReadLine(); + if (line != null && line.StartsWith(OperationSectionPrefix)) + { + return GetCardName(line); + } + } + + return null; + } + + private string GetCardName(string sectionName) + { + return sectionName.Replace(OperationSectionPrefix, string.Empty); } } - -} \ No newline at end of file +} diff --git a/PriorBankParser/PriorBankParser.csproj b/PriorBankParser/PriorBankParser.csproj new file mode 100644 index 0000000..d82ecd8 --- /dev/null +++ b/PriorBankParser/PriorBankParser.csproj @@ -0,0 +1,16 @@ + + + + netcoreapp3.1 + + + + + + + + + + + + diff --git a/PriorBankParser/SectionParseResult.cs b/PriorBankParser/SectionParseResult.cs deleted file mode 100644 index bc4f301..0000000 --- a/PriorBankParser/SectionParseResult.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System.Collections.Generic; - -namespace PriorBankParser -{ - public class SectionParseResultDto - { - public string ContractNo { get; set; } - - public IReadOnlyCollection Transactions { get; set; } - } -} \ No newline at end of file diff --git a/PriorBankParser/StatementDto.cs b/PriorBankParser/StatementDto.cs deleted file mode 100644 index b2e48fe..0000000 --- a/PriorBankParser/StatementDto.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System.Collections.Generic; - -namespace PriorBankParser -{ - public class StatementDto - { - public ICollection> AccountInfos { get; set; } - - public ICollection ContractsInfo { get; set; } - - public ICollection LockedInfo { get; set; } - - public TotalDto StatementTotal { get; set; } - } - - public class LockedSection - { - public string SectionName { get; set; } - - public IReadOnlyCollection Items { get; set; } - } - - public class OperationalSectionDto - { - public string SectionName { get; set; } - - public IReadOnlyCollection Items { get; set; } - - public TotalDto Total { get; set; } - } - - public class TotalDto - { - public decimal IncomeAmount { get; set; } - - public decimal OutcomeAmount { get; set; } - - public decimal Commission { get; set; } - - public decimal FinalAmount { get; set; } - } -} \ No newline at end of file diff --git a/PriorBankParser/TransactionInfoDto.cs b/PriorBankParser/TransactionDto.cs similarity index 55% rename from PriorBankParser/TransactionInfoDto.cs rename to PriorBankParser/TransactionDto.cs index 8edaa8f..52f82bc 100644 --- a/PriorBankParser/TransactionInfoDto.cs +++ b/PriorBankParser/TransactionDto.cs @@ -1,12 +1,16 @@ using System; -namespace PriorBankParser +namespace PriorBankParser.Dtos { - public class TransactionInfoDto + public class TransactionDto { + public string Contract { get; set; } + public DateTime TransactionDate { get; set; } - public string OperationName { get; set; } + public string TransactionName { get; set; } + + public string Category { get; set; } public decimal Amount { get; set; } @@ -18,8 +22,6 @@ namespace PriorBankParser public decimal AccountTurnover { get; set; } - public string DigitalCard { get; set; } - - public string OperationCategory { get; set; } + public bool IsIncome => Amount > 0 || Commission > 0; } -} \ No newline at end of file +} diff --git a/PriorBankParser/TransactionInfoDtoReadMap.cs b/PriorBankParser/TransactionInfoDtoReadMap.cs new file mode 100644 index 0000000..3c3303d --- /dev/null +++ b/PriorBankParser/TransactionInfoDtoReadMap.cs @@ -0,0 +1,21 @@ +using CsvHelper.Configuration; +using PriorBankParser.Dtos; +using static PriorBankParser.Constants.SourceColumns; + +namespace PriorBankParser +{ + internal sealed class TransactionInfoDtoReadMap : ClassMap + { + public TransactionInfoDtoReadMap() + { + Map(m => m.TransactionDate).Name(TransactionDate); + Map(m => m.TransactionName).Name(Operation); + Map(m => m.Amount).Name(Amount).WithDecimalConversion(); + Map(m => m.Currency).Name(Currency); + Map(m => m.OperationDate).Name(OperationDate); + Map(m => m.Commission).Name(Commission).WithDecimalConversion(); + Map(m => m.AccountTurnover).Name(AccountTurnover).WithDecimalConversion(); + Map(m => m.Category).Name(Category); + } + } +} diff --git a/PriorBankParser/TransactionInfoDtoWriteMap.cs b/PriorBankParser/TransactionInfoDtoWriteMap.cs new file mode 100644 index 0000000..4e1cc4c --- /dev/null +++ b/PriorBankParser/TransactionInfoDtoWriteMap.cs @@ -0,0 +1,45 @@ +using System.Globalization; +using CsvHelper.Configuration; +using PriorBankParser.Dtos; + +namespace PriorBankParser +{ + internal sealed class TransactionInfoDtoWriteMap : ClassMap + { + private const string DateTimeString = "yyyy-MM-dd hh:mm:ss"; + private const string NumberFormat = "F"; + + public TransactionInfoDtoWriteMap() + { + Map(m => m.Contract) + .ConvertUsing(m => m.Contract.Trim()) + .Index(10); + Map(m => m.TransactionName) + .ConvertUsing(m => m.TransactionName.Trim()) + .Index(20); + Map(m => m.Category) + .ConvertUsing(m => m.Category.Trim()) + .Index(30); + Map(m => m.Currency) + .ConvertUsing(m => m.Currency.Trim()) + .Index(40); + Map(m => m.IsIncome) + .Index(50); + Map(m => m.TransactionDate) + .Index(60) + .ConvertUsing(c => c.TransactionDate.ToString(DateTimeString, CultureInfo.InvariantCulture)); + Map(m => m.OperationDate) + .Index(70) + .ConvertUsing(c => c.OperationDate.ToString(DateTimeString, CultureInfo.InvariantCulture)); + Map(m => m.Amount) + .Index(80) + .ConvertUsing(c => c.Amount.ToString(NumberFormat, CultureInfo.InvariantCulture)); + Map(m => m.Commission) + .Index(90) + .ConvertUsing(c => c.Commission.ToString(NumberFormat, CultureInfo.InvariantCulture)); + Map(m => m.AccountTurnover) + .Index(100) + .ConvertUsing(c => c.AccountTurnover.ToString(NumberFormat, CultureInfo.InvariantCulture)); + } + } +} diff --git a/PriorBankParser/download_receipt.js b/PriorBankParser/download_receipt.js new file mode 100644 index 0000000..36ed9b9 --- /dev/null +++ b/PriorBankParser/download_receipt.js @@ -0,0 +1,26 @@ +function downloadCSVFile() { + + const data = [...document.querySelectorAll(".k-grid-content table[role='grid'] tbody tr")] + .map(row => [...row.querySelectorAll("td, th")] + .map(col => col.innerText) + .reduce((a, b) => + `${a},${b}`)).reduce((a, b) => `${a}\r\n${b}`); + + let fileName = document.querySelector('.filter-text span').innerHTML + .replace('за период c ', 'receipts_') + .replace(' по ', '_') + .replaceAll('.', '-'); + + fileName = fileName + '.csv' + + const csv_file = new Blob([data], {type: "text/csv"}); + const download_link = document.createElement("a"); + + download_link.download = fileName; + download_link.href = window.URL.createObjectURL(csv_file); + download_link.style.display = "none"; + document.body.appendChild(download_link); + download_link.click(); + + return fileName; +} diff --git a/TransactionListParser.sln b/TransactionListParser.sln new file mode 100644 index 0000000..3e62b3d --- /dev/null +++ b/TransactionListParser.sln @@ -0,0 +1,16 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PriorBankParser", "PriorBankParser\PriorBankParser.csproj", "{D855386F-0215-4DB0-9BF1-6542672009DB}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D855386F-0215-4DB0-9BF1-6542672009DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D855386F-0215-4DB0-9BF1-6542672009DB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D855386F-0215-4DB0-9BF1-6542672009DB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D855386F-0215-4DB0-9BF1-6542672009DB}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal