my stop words contain
this words:
من
and my input text:
ايمن
من
my output:
اي [1]
my problem:
Ignore the word connected to the characters
ايمن
after filter
ايمن [1]
not
اي [1]
im use this code
------------------------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Text;
using DAL;
public partial class _Default : System.Web.UI.Page
{
private static char[] delimiters_no_digits = new char[] {
'{', '}', '(', ')', '[', ']', '>', '<','-', '_', '=', '+',
'|', '\\', ':', ';', ' ', ',', '.', '/', '?', '~', '!',
'@', '#', '$', '%', '^', '&', '*', ' ', '\r', '\n', '\t',
'"' ,'0', '1', '2', '3', '4', '5', '6', '7', '8', '9' ,'?'
};
private static List<string> delimiters_no_digits2 = null;
protected void Page_Load(object sender, EventArgs e)
{
//if (!IsPostBack)
//{
delimiters_no_digits2 = new List<string>();
string query = " select WORD , STEMMEDWORD from STOPWORDS ";
using (System.Data.SqlClient.SqlCommand cmd = DBConnection.GetSqlConnection().CreateCommand())
{
cmd.CommandText = query;
cmd.CommandType = System.Data.CommandType.Text;
cmd.Connection.Open();
using (var dr = cmd.ExecuteReader())
{
while (dr.Read())
{
delimiters_no_digits2.Add(dr["WORD"].ToString());
// delimiters_no_digits2.Add(dr["STEMMEDWORD"].ToString());
}
}
cmd.Connection.Close();
}
// }
}
protected void btnGo_Click(object sender, EventArgs e)
{
textBoxOutput.Text = string.Empty;
if (textBoxInput.Text != string.Empty)
{
string[] words = Tokenize(textBoxInput.Text);
if (words.Length > 0)
{
SortedDictionary<string, int> dict = new SortedDictionary<string, int>();
foreach (string word in words)
{
if (dict.ContainsKey(word))
{
dict[word]++;
}
else
{
dict.Add(word, 1);
}
}
StringBuilder resultSb = new StringBuilder(dict.Count * 9);
foreach (KeyValuePair<string, int> entry in dict)
{
resultSb.AppendLine(string.Format("{0} [{1}]", entry.Key, entry.Value));
}
textBoxOutput.Text = resultSb.ToString();
}
}
}
public static string[] Tokenize(string text)
{
string[] tokens2 = text.Split(delimiters_no_digits2.ToArray(), StringSplitOptions.RemoveEmptyEntries);
string[] tokens1 = string.Join("", tokens2).ToString().Split(delimiters_no_digits, StringSplitOptions.RemoveEmptyEntries);
var ars = new List<string>();
ars.AddRange(tokens1);
string[] tokens = ars.ToArray();
for (int i = 0; i < tokens.Length; i++)
{
string token = tokens;
// Change token only when it starts and/or ends with "'" and
// it has at least 2 characters.
if (token.Length > 1)
{
if (token.StartsWith("'") && token.EndsWith("'"))
tokens = token.Substring(1, token.Length - 2); // remove the starting and ending "'"
else if (token.StartsWith("'"))
tokens = token.Substring(1); // remove the starting "'"
else if (token.EndsWith("'"))
tokens = token.Substring(0, token.Length - 1); // remove the last "'"
}
}
return tokens;
}
}
يرفع من قيمة الموضوع واهمية الرد