[dotnet] Add dotnet projects and examples
+ Sitemap generator I created while learning the dispose pattern + Testing project for learning general C#
This commit is contained in:
114
dotnet/sitemap/SiteMapLibrary/SiteMap.cs
Normal file
114
dotnet/sitemap/SiteMapLibrary/SiteMap.cs
Normal file
@@ -0,0 +1,114 @@
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace SiteMapLibrary;
|
||||
|
||||
public class SiteMap : IDisposable
|
||||
{
|
||||
private HttpClient _client;
|
||||
private HashSet<string> _foundUrls;
|
||||
private HashSet<string> _visitedUrls;
|
||||
private Queue<string> _visitQueue;
|
||||
private bool _disposed = false;
|
||||
private XmlManager XmlManager { get; set; }
|
||||
public string? Url { get; private set; }
|
||||
public Regex Regexp { get; set; }
|
||||
|
||||
public SiteMap(string url, string savepath, Regex pattern)
|
||||
{
|
||||
Url = url;
|
||||
_client = new HttpClient();
|
||||
_foundUrls = new HashSet<string>();
|
||||
_visitedUrls = new HashSet<string>();
|
||||
_visitQueue = new Queue<string>();
|
||||
Regexp = pattern;
|
||||
XmlManager = new XmlManager(savepath);
|
||||
}
|
||||
|
||||
public SiteMap(string url, XmlManager mgr, Regex pattern)
|
||||
{
|
||||
_client = new HttpClient();
|
||||
_foundUrls = new HashSet<string>();
|
||||
_visitedUrls = new HashSet<string>();
|
||||
_visitQueue = new Queue<string>();
|
||||
Regexp = pattern;
|
||||
Url = url;
|
||||
XmlManager = mgr;
|
||||
}
|
||||
|
||||
public async Task Crawl()
|
||||
{
|
||||
while (Url != null)
|
||||
{
|
||||
_visitedUrls.Add(Url);
|
||||
using var content = await _client.GetAsync(Url);
|
||||
if (!content.IsSuccessStatusCode)
|
||||
{
|
||||
Console.WriteLine($"{content.StatusCode} on url: {Url}");
|
||||
NextUrl();
|
||||
continue;
|
||||
}
|
||||
|
||||
var m = Regexp.Match(await content.Content.ReadAsStringAsync());
|
||||
while (m.Success)
|
||||
{
|
||||
foreach (Group group in m.Groups)
|
||||
{
|
||||
if (_foundUrls.Add(group.Value))
|
||||
{
|
||||
Console.WriteLine(group.Value);
|
||||
// Console.WriteLine(content.Content.Headers.LastModified);
|
||||
if (!_visitedUrls.Contains(group.Value) && !_visitQueue.Contains(group.Value))
|
||||
{
|
||||
_visitQueue.Enqueue(group.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m = m.NextMatch();
|
||||
}
|
||||
|
||||
NextUrl();
|
||||
content.Dispose();
|
||||
}
|
||||
WriteXml();
|
||||
}
|
||||
|
||||
private void WriteXml()
|
||||
{
|
||||
List<string> urls = new List<string>(_visitedUrls.OrderBy(k => k.Length).ToArray());
|
||||
foreach (string url in urls)
|
||||
{
|
||||
XmlManager.AddUrl(url);
|
||||
}
|
||||
XmlManager.Save();
|
||||
}
|
||||
|
||||
private void NextUrl()
|
||||
{
|
||||
if (_visitQueue.Count == 0)
|
||||
{
|
||||
Url = null;
|
||||
return;
|
||||
}
|
||||
Url = _visitQueue.Dequeue();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
public void Dispose(bool disposing)
|
||||
{
|
||||
if (!_disposed)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
_client.Dispose();
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
10
dotnet/sitemap/SiteMapLibrary/SiteMapLibrary.csproj
Normal file
10
dotnet/sitemap/SiteMapLibrary/SiteMapLibrary.csproj
Normal file
@@ -0,0 +1,10 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net6.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<RootNamespace>SiteMap</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
59
dotnet/sitemap/SiteMapLibrary/XmlManager.cs
Normal file
59
dotnet/sitemap/SiteMapLibrary/XmlManager.cs
Normal file
@@ -0,0 +1,59 @@
|
||||
using System.Xml;
|
||||
|
||||
namespace SiteMapLibrary;
|
||||
|
||||
public class XmlManager
|
||||
{
|
||||
private XmlDocument XmlDocument { get; set; }
|
||||
private XmlDeclaration XmlDeclaration { get; set; }
|
||||
private XmlElement XmlUrlset { get; set; }
|
||||
private string Path { get; set; }
|
||||
|
||||
public XmlManager(string path,
|
||||
string version="1.0", string encoding="utf-8", string standalone="")
|
||||
{
|
||||
XmlDocument = new XmlDocument();
|
||||
XmlDeclaration = XmlDocument.CreateXmlDeclaration(version, encoding, standalone);
|
||||
XmlDocument.AppendChild(XmlDeclaration);
|
||||
XmlUrlset = XmlDocument.CreateElement("urlset");
|
||||
XmlDocument.AppendChild(XmlUrlset);
|
||||
Path = path;
|
||||
}
|
||||
|
||||
~XmlManager()
|
||||
{
|
||||
Save();
|
||||
}
|
||||
|
||||
public void AddUrl(string url)
|
||||
{
|
||||
XmlElement newUrl = XmlDocument.CreateElement("url");
|
||||
XmlUrlset.AppendChild(newUrl);
|
||||
XmlElement newLoc = XmlDocument.CreateElement("loc");
|
||||
newLoc.InnerText = url;
|
||||
newUrl.AppendChild(newLoc);
|
||||
var lastmod = XmlDocument.CreateElement("lastmod");
|
||||
lastmod.InnerText = DateTime.Now.Year.ToString()
|
||||
+ '-' + DateTime.Now.Month.ToString()
|
||||
+ '-' + DateTime.Now.Day;
|
||||
newUrl.AppendChild(lastmod);
|
||||
var changeFreq = XmlDocument.CreateElement("changefreq");
|
||||
changeFreq.InnerText = "daily";
|
||||
newUrl.AppendChild(changeFreq);
|
||||
var priority = XmlDocument.CreateElement("priority");
|
||||
priority.InnerText = "0.5";
|
||||
newUrl.AppendChild(priority);
|
||||
}
|
||||
|
||||
public void Save()
|
||||
{
|
||||
if (Path == "Console.Out")
|
||||
{
|
||||
XmlDocument.Save(Console.Out);
|
||||
}
|
||||
else
|
||||
{
|
||||
XmlDocument.Save(Path);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user