me/transactions-list-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial setup - gitea

Browse Source
This commit is contained in:
Vladimir K
2023-07-23 17:10:20 +03:00
parent f9c957506c
commit 232a59569c
12 changed files with 370 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
PriorBankParser/Constants.cs
View File

@@ -1,11 +1,65 @@
namespace PriorBankParser
using System.Globalization;
using CsvHelper.Configuration;
namespace PriorBankParser
{
public static class Constants
internal static class Constants
{
public static class SectionNames
internal static class SourceConfig
{
internal const string CodePage = "windows-1251";
internal const string CultureInfoName = "ru-RU";
internal static CultureInfo SrcCultureInfo { get; } = new CultureInfo(CultureInfoName);
internal static CsvConfiguration SrcCsvConfiguration { get; } = GetReadConfig();
internal const string CsvDelimiter = ";";
}
private static CsvConfiguration GetReadConfig()
{
var config = new CsvConfiguration(SourceConfig.SrcCultureInfo)
{
Delimiter = SourceConfig.CsvDelimiter,
IgnoreBlankLines = true,
IgnoreQuotes = true,
};
config.RegisterClassMap<TransactionInfoDtoReadMap>();
return config;
}
public static CsvConfiguration GetWriteConfig()
{
var config = new CsvConfiguration(CultureInfo.InvariantCulture)
{
Delimiter = ",",
ShouldQuote = (s, context) => true,
};
config.RegisterClassMap<TransactionInfoDtoWriteMap>();
return config;
}
internal static class SourceColumns
{
internal const string TransactionDate = "Дата транзакции";
internal const string Operation = "Операция";
internal const string Amount = "Сумма";
internal const string Currency = "Валюта";
internal const string OperationDate = "Дата операции по счету";
internal const string Commission = "Комиссия/Money-back";
internal const string AccountTurnover = "Обороты по счету";
internal const string Category = "Категория операции";
}
internal static class SectionNames
{
internal const string LockedSectionPrefix = "Заблокированные суммы по ........";
internal const string OperationSectionPrefix = "Операции по ........";
internal const string CardTotalSectionPrefix = "Всего по контракту";
internal const string StatementTotalSectionPrefix = "Всего в данной валюте";
}
}
}

61
PriorBankParser/DbLoader.cs Normal file
View File

@@ -0,0 +1,61 @@
using System.Data.SqlClient;
using System.Linq;
using Dapper;
using NUnit.Framework;
namespace PriorBankParser
{
public class DbLoader
{
private const string Sql = @"
DROP TABLE IF EXISTS [dbo].[VPSK_IMPORT]
GO
CREATE TABLE [dbo].[VPSK_IMPORT]
(
[Contract] NVARCHAR(255),
[TransactionName] NVARCHAR(2048),
[Category] NVARCHAR(2048),
[Currency] VARCHAR(5),
[IsIncome] VARCHAR(5),
[TransactionDate] DATETIME,
[OperationDate] DATETIME,
[Amount] DECIMAL(19, 6),
[Commission] DECIMAL(19, 6),
[AccountTurnover] DECIMAL(19, 6)
)
GO
BULK INSERT [dbo].[VPSK_IMPORT]
FROM '<SOURCE_FILE_PATH>'
WITH (
TABLOCK
,FORMAT = 'CSV'
,FIRSTROW = 2
,FIELDTERMINATOR = ','
,FIELDQUOTE = '0x22'
--,ROWTERMINATOR = '0x0a'
,CODEPAGE=65001
);
";
[Test]
public void Load()
{
const string sourceFile = @"D:\Cloud-storage\Dropbox\Home\Finance\Raw Source\ConvertedResult.csv";
var script = Sql.Replace("<SOURCE_FILE_PATH>", sourceFile);
var connectionString = new SqlConnectionStringBuilder
{ DataSource = "(local)", InitialCatalog = "MY_FINANCE", IntegratedSecurity = true, }
.ConnectionString;
using var sqlConnection = new SqlConnection(connectionString);
sqlConnection.Open();
foreach (var scriptPart in script.Split("GO").Where(s => !string.IsNullOrEmpty(s)))
{
sqlConnection.Execute(scriptPart, commandTimeout: 60 * 60);
}
}
}
}

14
PriorBankParser/MemberMapX.cs Normal file
View File

@@ -0,0 +1,14 @@
using System.Globalization;
using CsvHelper.Configuration;
namespace PriorBankParser
{
public static class MemberMapX
{
public static MemberMap WithDecimalConversion(this MemberMap map)
{
return map.TypeConverterOption.NumberStyles(NumberStyles.Any)
.TypeConverterOption.CultureInfo(Constants.SourceConfig.SrcCultureInfo);
}
}
}

167
PriorBankParser/Parser.cs
View File

@@ -1,83 +1,124 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Globalization;
using System.Collections.Generic;
using System.IO;
using System.Text;
using CsvHelper;
using CsvHelper.Configuration;
using NUnit.Framework;
using PriorBankParser.Dtos;
using static PriorBankParser.Constants.SectionNames;
using static PriorBankParser.Constants.SourceConfig;
namespace PriorBankParser
{
public class SectionParser
public class Parser
{
public enum SectionType
[Test]
public void Run()
{
AccountInfo,
StatementDetails,
ContractOperations,
ContractLocked,
ContractTotal,
StatementTotal
const string sourceDirectoryPath = @"D:\Cloud-storage\Dropbox\Home\Finance\Raw Source\";
const string resultFileName = @"ConvertedResult.csv";
var result = ParseFilesInDirectory(sourceDirectoryPath);
var resultFilePath = Path.Combine(sourceDirectoryPath, resultFileName);
WriteResult(resultFilePath, result);
}
public SectionParseResultDto Parse(string filePath)
private ICollection<TransactionDto> ParseFilesInDirectory(string directoryPath)
{
var result = new SectionParseResultDto();
var transactions = new List<TransactionInfoDto>();
result.Transactions = transactions;
var files = Directory.GetFiles(directoryPath, "*.csv");
var result = new List<TransactionDto>();
using var reader = new StreamReader(filePath);
using var csv = new CsvReader(reader, CultureInfo.InvariantCulture);
csv.Configuration.HasHeaderRecord = false;
csv.Configuration.RegisterClassMap<TransactionInfoDtoMap>();
//csv.Configuration.
var currentSection = SectionType.AccountInfo;
var contractName = string.Empty;
while (csv.Read())
foreach (var file in files)
{
var field = csv.GetField(0);
if (field.Equals(Constants.SectionNames.OperationSectionPrefix))
{
currentSection = SectionType.StatementTotal;
contractName = field.Replace(Constants.SectionNames.OperationSectionPrefix, string.Empty);
continue;
}
//switch (currentSection)
//{
// case "FooId":
// fooRecords.Add(csv.GetRecord<Foo>());
// break;
// case "BarId":
// barRecords.Add(csv.GetRecord<Bar>());
// break;
// default:
// throw new InvalidOperationException("Unknown record type.");
//}
result.AddRange(ParseFile(file));
}
return result;
}
}
public sealed class TransactionInfoDtoMap : ClassMap<TransactionInfoDto>
{
public TransactionInfoDtoMap()
private ICollection<TransactionDto> ParseFile(string filePath)
{
Map(m => m.TransactionDate).Name("Дата транзакции");
Map(m => m.OperationName).Name("Операция");
Map(m => m.Amount).Name("Сумма");
Map(m => m.Currency).Name("Валюта");
Map(m => m.OperationDate).Name("Дата операции по счету");
Map(m => m.Commission).Name("Комиссия/Money-back");
Map(m => m.AccountTurnover).Name("Обороты по счету");
Map(m => m.DigitalCard).Name("Цифровая карта");
Map(m => m.OperationCategory).Name("Категория операции");
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
var result = new List<TransactionDto>();
using var reader = new StreamReader(filePath, Encoding.GetEncoding(CodePage));
var contractName = SeekToTheTransactions(reader);
if (string.IsNullOrEmpty(contractName))
{
return result;
}
using var csvReader = new CsvReader(reader, SrcCsvConfiguration);
var isHeader = true;
var ignore = false;
while (csvReader.Read())
{
var field = csvReader.GetField(0);
if (field.Equals(StatementTotalSectionPrefix))
{
break;
}
if (field.Equals(CardTotalSectionPrefix))
{
ignore = true;
}
if (field.StartsWith(OperationSectionPrefix))
{
contractName = GetCardName(field);
isHeader = true;
continue;
}
if (isHeader)
{
csvReader.ReadHeader();
isHeader = false;
ignore = false;
continue;
}
if (ignore)
{
continue;
}
var record = csvReader.GetRecord<TransactionDto>();
record.Contract = contractName;
result.Add(record);
}
return result;
}
private void WriteResult(string resultFilePath, IEnumerable<TransactionDto> records)
{
using var writer = new StreamWriter(resultFilePath, false, Encoding.UTF8);
using var csv = new CsvWriter(writer, Constants.GetWriteConfig());
csv.WriteRecords(records);
}
private string SeekToTheTransactions(StreamReader reader)
{
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
if (line != null && line.StartsWith(OperationSectionPrefix))
{
return GetCardName(line);
}
}
return null;
}
private string GetCardName(string sectionName)
{
return sectionName.Replace(OperationSectionPrefix, string.Empty);
}
}
}
}

16
PriorBankParser/PriorBankParser.csproj Normal file
View File

@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CsvHelper" Version="15.0.0" />
<PackageReference Include="Dapper" Version="2.0.90" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
<PackageReference Include="NUnit" Version="3.12.0" />
<PackageReference Include="NUnit3TestAdapter" Version="3.16.1" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.2" />
</ItemGroup>
</Project>

11
PriorBankParser/SectionParseResult.cs
View File

@@ -1,11 +0,0 @@
using System.Collections.Generic;
namespace PriorBankParser
{
public class SectionParseResultDto
{
public string ContractNo { get; set; }
public IReadOnlyCollection<TransactionInfoDto> Transactions { get; set; }
}
}

42
PriorBankParser/StatementDto.cs
View File

@@ -1,42 +0,0 @@
using System.Collections.Generic;
namespace PriorBankParser
{
public class StatementDto
{
public ICollection<KeyValuePair<string, string>> AccountInfos { get; set; }
public ICollection<OperationalSectionDto> ContractsInfo { get; set; }
public ICollection<LockedSection> LockedInfo { get; set; }
public TotalDto StatementTotal { get; set; }
}
public class LockedSection
{
public string SectionName { get; set; }
public IReadOnlyCollection<TransactionInfoDto> Items { get; set; }
}
public class OperationalSectionDto
{
public string SectionName { get; set; }
public IReadOnlyCollection<TransactionInfoDto> Items { get; set; }
public TotalDto Total { get; set; }
}
public class TotalDto
{
public decimal IncomeAmount { get; set; }
public decimal OutcomeAmount { get; set; }
public decimal Commission { get; set; }
public decimal FinalAmount { get; set; }
}
}

16
PriorBankParser/TransactionInfoDto.cs → PriorBankParser/TransactionDto.cs
View File

@@ -1,12 +1,16 @@
using System;
namespace PriorBankParser
namespace PriorBankParser.Dtos
{
public class TransactionInfoDto
public class TransactionDto
{
public string Contract { get; set; }
public DateTime TransactionDate { get; set; }
public string OperationName { get; set; }
public string TransactionName { get; set; }
public string Category { get; set; }
public decimal Amount { get; set; }
@@ -18,8 +22,6 @@ namespace PriorBankParser
public decimal AccountTurnover { get; set; }
public string DigitalCard { get; set; }
public string OperationCategory { get; set; }
public bool IsIncome => Amount > 0 || Commission > 0;
}
}
}

21
PriorBankParser/TransactionInfoDtoReadMap.cs Normal file
View File

@@ -0,0 +1,21 @@
using CsvHelper.Configuration;
using PriorBankParser.Dtos;
using static PriorBankParser.Constants.SourceColumns;
namespace PriorBankParser
{
internal sealed class TransactionInfoDtoReadMap : ClassMap<TransactionDto>
{
public TransactionInfoDtoReadMap()
{
Map(m => m.TransactionDate).Name(TransactionDate);
Map(m => m.TransactionName).Name(Operation);
Map(m => m.Amount).Name(Amount).WithDecimalConversion();
Map(m => m.Currency).Name(Currency);
Map(m => m.OperationDate).Name(OperationDate);
Map(m => m.Commission).Name(Commission).WithDecimalConversion();
Map(m => m.AccountTurnover).Name(AccountTurnover).WithDecimalConversion();
Map(m => m.Category).Name(Category);
}
}
}

45
PriorBankParser/TransactionInfoDtoWriteMap.cs Normal file
View File

@@ -0,0 +1,45 @@
using System.Globalization;
using CsvHelper.Configuration;
using PriorBankParser.Dtos;
namespace PriorBankParser
{
internal sealed class TransactionInfoDtoWriteMap : ClassMap<TransactionDto>
{
private const string DateTimeString = "yyyy-MM-dd hh:mm:ss";
private const string NumberFormat = "F";
public TransactionInfoDtoWriteMap()
{
Map(m => m.Contract)
.ConvertUsing(m => m.Contract.Trim())
.Index(10);
Map(m => m.TransactionName)
.ConvertUsing(m => m.TransactionName.Trim())
.Index(20);
Map(m => m.Category)
.ConvertUsing(m => m.Category.Trim())
.Index(30);
Map(m => m.Currency)
.ConvertUsing(m => m.Currency.Trim())
.Index(40);
Map(m => m.IsIncome)
.Index(50);
Map(m => m.TransactionDate)
.Index(60)
.ConvertUsing(c => c.TransactionDate.ToString(DateTimeString, CultureInfo.InvariantCulture));
Map(m => m.OperationDate)
.Index(70)
.ConvertUsing(c => c.OperationDate.ToString(DateTimeString, CultureInfo.InvariantCulture));
Map(m => m.Amount)
.Index(80)
.ConvertUsing(c => c.Amount.ToString(NumberFormat, CultureInfo.InvariantCulture));
Map(m => m.Commission)
.Index(90)
.ConvertUsing(c => c.Commission.ToString(NumberFormat, CultureInfo.InvariantCulture));
Map(m => m.AccountTurnover)
.Index(100)
.ConvertUsing(c => c.AccountTurnover.ToString(NumberFormat, CultureInfo.InvariantCulture));
}
}
}

26
PriorBankParser/download_receipt.js Normal file
View File

@@ -0,0 +1,26 @@
function downloadCSVFile() {
const data = [...document.querySelectorAll(".k-grid-content table[role='grid'] tbody tr")]
.map(row => [...row.querySelectorAll("td, th")]
.map(col => col.innerText)
.reduce((a, b) =>
`${a},${b}`)).reduce((a, b) => `${a}\r\n${b}`);
let fileName = document.querySelector('.filter-text span').innerHTML
.replace('за период c ', 'receipts_')
.replace(' по ', '_')
.replaceAll('.', '-');
fileName = fileName + '.csv'
const csv_file = new Blob([data], {type: "text/csv"});
const download_link = document.createElement("a");
download_link.download = fileName;
download_link.href = window.URL.createObjectURL(csv_file);
download_link.style.display = "none";
document.body.appendChild(download_link);
download_link.click();
return fileName;
}

16
TransactionListParser.sln Normal file
View File

@@ -0,0 +1,16 @@
Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PriorBankParser", "PriorBankParser\PriorBankParser.csproj", "{D855386F-0215-4DB0-9BF1-6542672009DB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{D855386F-0215-4DB0-9BF1-6542672009DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D855386F-0215-4DB0-9BF1-6542672009DB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D855386F-0215-4DB0-9BF1-6542672009DB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D855386F-0215-4DB0-9BF1-6542672009DB}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal